From d3818ea20c092d791082e40352dba756079a7bf4 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Tue, 22 Oct 2024 15:47:33 +0200 Subject: [PATCH 01/18] Handle async responses asynchronously. This improves indexing time, especially when the ES connection has a high latency or the ES load is high. --- .../geonet/kernel/search/EsSearchManager.java | 75 ++++++++++--------- .../org/fao/geonet/index/es/EsRestClient.java | 33 ++------ .../worker/EsWFSFeatureIndexer.java | 11 +-- 3 files changed, 44 insertions(+), 75 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 978ab63a750..169d5902075 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -67,6 +67,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -87,6 +88,8 @@ public class EsSearchManager implements ISearchManager { public static final String FIELDNAME = "name"; public static final String FIELDSTRING = "string"; + private static final int COMMIT_INTERVAL = 200; + public static final Map RELATED_INDEX_FIELDS; public static final Set FIELDLIST_CORE; public static final Set FIELDLIST_RELATED; @@ -173,11 +176,7 @@ public void setIndexType(String indexType) { @Autowired private OverviewIndexFieldUpdater overviewFieldUpdater; - private int commitInterval = 200; - - // public for test, to be private or protected - public Map listOfDocumentsToIndex = - Collections.synchronizedMap(new HashMap<>()); + private final Map listOfDocumentsToIndex = Collections.synchronizedMap(new HashMap<>()); private Map indexList; private Path getXSLTForIndexing(Path schemaDir, MetadataType metadataType) { @@ -397,7 +396,7 @@ public void updateFieldsAsynch(String id, Map fields) { .doc(fields) ); - client.getAsynchClient() + client.getAsyncClient() .update(updateRequest, ObjectNode.class) .whenComplete((response, exception) -> { if (exception != null) { @@ -453,37 +452,38 @@ public void index(Path schemaDir, Element metadata, String id, String jsonDocument = mapper.writeValueAsString(doc); - if (forceRefreshReaders) { - Map document = new HashMap<>(); - document.put(id, jsonDocument); - final BulkResponse bulkItemResponses = client.bulkRequest(defaultIndex, document); - checkIndexResponse(bulkItemResponses, document); - overviewFieldUpdater.process(id); - } else { - listOfDocumentsToIndex.put(id, jsonDocument); - if (listOfDocumentsToIndex.size() == commitInterval) { - sendDocumentsToIndex(); - } + listOfDocumentsToIndex.put(id, jsonDocument); + if (forceRefreshReaders || listOfDocumentsToIndex.size() >= COMMIT_INTERVAL) { + sendDocumentsToIndex(); } } private void sendDocumentsToIndex() { - Map documents = new HashMap<>(listOfDocumentsToIndex); - listOfDocumentsToIndex.clear(); - if (!documents.isEmpty()) { - try { - final BulkResponse bulkItemResponses = client - .bulkRequest(defaultIndex, documents); - checkIndexResponse(bulkItemResponses, documents); - } catch (Exception e) { - LOGGER.error( - "An error occurred while indexing {} documents in current indexing list. Error is {}.", - listOfDocumentsToIndex.size(), e.getMessage()); - } finally { - // TODO: Trigger this async ? - documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid)); - } + Map documents; + synchronized (listOfDocumentsToIndex) { + if (listOfDocumentsToIndex.isEmpty()) return; + documents = new HashMap<>(listOfDocumentsToIndex); + listOfDocumentsToIndex.clear(); } + + BulkRequest bulkRequest = client.buildBulkRequest(defaultIndex, documents); + client.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + try { + checkIndexResponse(bulkItemResponses, documents); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid)); + } + } + ) + .exceptionally(e -> { + LOGGER.error( + "An error occurred while indexing {} documents in current indexing list.", + documents.size(), e); + return null; + }); } private void checkIndexResponse(BulkResponse bulkItemResponses, @@ -537,7 +537,8 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, }); if (!listErrorOfDocumentsToIndex.isEmpty()) { - BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex); + BulkRequest bulkRequest = client.buildBulkRequest(defaultIndex, listErrorOfDocumentsToIndex); + BulkResponse response = client.getClient().bulk(bulkRequest); if (response.errors()) { LOGGER.error("Failed to save error documents {}.", Arrays.toString(errorDocumentIds.toArray())); @@ -546,9 +547,9 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, } } - private static ImmutableSet booleanFields; - private static ImmutableSet arrayFields; - private static ImmutableSet booleanValues; + private static final ImmutableSet booleanFields; + private static final ImmutableSet arrayFields; + private static final ImmutableSet booleanValues; static { arrayFields = ImmutableSet.builder() @@ -930,7 +931,7 @@ public static String analyzeField(String analyzer, } public boolean isIndexing() { - return listOfDocumentsToIndex.size() > 0; + return !listOfDocumentsToIndex.isEmpty(); } public boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException { diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 4b21f7c61a3..1f1b6be2507 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -31,7 +31,6 @@ import co.elastic.clients.elasticsearch._types.query_dsl.WrapperQuery; import co.elastic.clients.elasticsearch.cluster.HealthResponse; import co.elastic.clients.elasticsearch.core.*; -import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.AnalyzeRequest; import co.elastic.clients.elasticsearch.indices.AnalyzeResponse; import co.elastic.clients.elasticsearch.indices.analyze.AnalyzeToken; @@ -110,7 +109,7 @@ public ElasticsearchClient getClient() { return client; } - public ElasticsearchAsyncClient getAsynchClient() { + public ElasticsearchAsyncClient getAsyncClient() { return asyncClient; } @@ -208,11 +207,9 @@ public EsRestClient setPassword(String password) { return this; } - public static final String ROUTING_KEY = "101"; - - public BulkResponse bulkRequest(String index, Map docs) throws IOException { + public BulkRequest buildBulkRequest(String index, Map docs) { if (!activated) { - throw new IOException("Index not yet activated."); + throw new IllegalStateException("Index not yet activated."); } BulkRequest.Builder requestBuilder = new BulkRequest.Builder() @@ -222,10 +219,7 @@ public BulkResponse bulkRequest(String index, Map docs) throws I JsonpMapper jsonpMapper = client._transport().jsonpMapper(); JsonProvider jsonProvider = jsonpMapper.jsonProvider(); - Iterator> iterator = docs.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - + for (Map.Entry entry : docs.entrySet()) { JsonData jd = JsonData.from(jsonProvider.createParser(new StringReader(entry.getValue())), jsonpMapper); requestBuilder @@ -234,26 +228,9 @@ public BulkResponse bulkRequest(String index, Map docs) throws I .document(jd))); } - BulkRequest request = requestBuilder.build(); - - try { - return client.bulk(request); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } + return requestBuilder.build(); } -// -// public void bulkRequestAsync(Bulk.Builder bulk , JestResultHandler handler) { -// client.executeAsync(bulk.build(), handler); -// -// } -// -// public BulkResult bulkRequestSync(Bulk.Builder bulk) throws IOException { -// return client.execute(bulk.build()); -// } - /** * Query using Lucene query syntax. diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java index 3d0857ddbb6..dcc1b25cbd1 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java @@ -30,10 +30,6 @@ import co.elastic.clients.elasticsearch.core.BulkResponse; import co.elastic.clients.elasticsearch.core.IndexRequest; import co.elastic.clients.elasticsearch.core.IndexResponse; -import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; -import co.elastic.clients.elasticsearch.core.bulk.IndexOperation; -import co.elastic.clients.json.JsonData; -import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.ContentType; import com.fasterxml.jackson.core.JsonProcessingException; @@ -43,7 +39,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; -import jakarta.json.spi.JsonProvider; import org.apache.camel.Exchange; import org.apache.camel.spring.SpringCamelContext; import org.apache.jcs.access.exception.InvalidArgumentException; @@ -74,7 +69,6 @@ import org.springframework.beans.factory.annotation.Value; import java.io.IOException; -import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -84,9 +78,6 @@ import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -634,7 +625,7 @@ public void afterBulk(long executionId, BulkRequest request, List contex } }; - this.bulk = BulkIngester.of(b -> b.client(client.getAsynchClient()) + this.bulk = BulkIngester.of(b -> b.client(client.getAsyncClient()) .listener(listener) // .maxConcurrentRequests(1) // .flushInterval(10, TimeUnit.SECONDS) From 2c089742678630731f69f10ffb94859b27c572b5 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Tue, 29 Oct 2024 16:28:19 +0100 Subject: [PATCH 02/18] Delegate the responsibility for batching to the caller --- .../org/fao/geonet/kernel/DataManager.java | 9 +- .../fao/geonet/kernel/IndexMetadataTask.java | 6 +- .../kernel/datamanager/IMetadataIndexer.java | 3 +- .../kernel/datamanager/IMetadataManager.java | 5 +- .../datamanager/base/BaseMetadataIndexer.java | 14 +- .../datamanager/base/BaseMetadataManager.java | 12 +- .../datamanager/base/BaseMetadataStatus.java | 7 +- .../datamanager/base/BaseMetadataUtils.java | 7 +- .../datamanager/draft/DraftMetadataUtils.java | 3 +- .../org/fao/geonet/kernel/mef/Importer.java | 3 +- .../geonet/kernel/search/EsSearchManager.java | 112 +++++------- .../geonet/kernel/search/ISearchManager.java | 6 +- .../index/BatchOpsMetadataReindexer.java | 26 +-- .../kernel/search/index/IndexingTask.java | 10 +- .../submission/BatchingIndexSubmittor.java | 100 +++++++++++ .../submission/DirectIndexSubmittor.java | 30 ++++ .../search/submission/IIndexSubmittor.java | 10 ++ .../geonet/AbstractCoreIntegrationTest.java | 12 +- ...ctIntegrationTestWithMockedSingletons.java | 3 +- .../kernel/DataManagerIntegrationTest.java | 2 +- .../geonet/kernel/url/UrlAnalyzerTest.java | 3 +- .../fao/geonet/component/csw/Transaction.java | 3 +- .../csw/CswTransactionIntegrationTest.java | 3 +- .../geonet/kernel/harvest/BaseAligner.java | 11 +- .../harvester/arcsde/ArcSDEHarvester.java | 47 ++--- .../kernel/harvest/harvester/csw/Aligner.java | 8 +- .../harvest/harvester/csw/Harvester.java | 5 +- .../harvester/fragment/FragmentHarvester.java | 10 +- .../harvest/harvester/geoPREST/Aligner.java | 6 +- .../harvest/harvester/geoPREST/Harvester.java | 6 +- .../harvest/harvester/geonet/Aligner.java | 8 +- .../harvest/harvester/geonet/Harvester.java | 5 +- .../harvest/harvester/geonet20/Aligner.java | 5 +- .../LocalFilesystemHarvester.java | 79 +++++---- .../LocalFsHarvesterFileVisitor.java | 7 +- .../harvest/harvester/oaipmh/Harvester.java | 8 +- .../harvester/oaipmh/OaiPmhHarvester.java | 5 +- .../harvest/harvester/ogcwxs/Harvester.java | 4 +- .../harvester/ogcwxs/OgcWxSHarvester.java | 5 +- .../harvest/harvester/simpleurl/Aligner.java | 4 +- .../harvester/simpleurl/Harvester.java | 167 +++++++++--------- .../harvest/harvester/thredds/Harvester.java | 5 +- .../harvester/thredds/ThreddsHarvester.java | 5 +- .../harvest/harvester/webdav/Harvester.java | 8 +- .../harvester/webdav/WebDavHarvester.java | 5 +- .../harvester/wfsfeatures/Harvester.java | 12 +- .../api/processing/DatabaseProcessUtils.java | 3 +- .../api/processing/XslProcessUtils.java | 3 +- .../api/records/MetadataInsertDeleteApi.java | 3 +- .../geonet/api/records/MetadataSampleApi.java | 3 +- .../api/records/MetadataSharingApi.java | 7 +- .../fao/geonet/api/records/MetadataUtils.java | 5 +- .../api/records/MetadataValidateApi.java | 3 +- .../api/records/MetadataWorkflowApi.java | 7 +- .../api/records/editing/AjaxEditUtils.java | 5 +- .../records/editing/MetadataEditingApi.java | 5 +- .../geonet/api/registries/DirectoryUtils.java | 5 +- .../api/selections/UserSelectionsApi.java | 20 ++- .../fao/geonet/api/links/LinksApiTest.java | 2 +- .../geonet/api/records/MetadataApiTest.java | 2 +- .../api/records/MetadataValidateApiTest.java | 5 +- .../api/records/MetadataWorkflowApiTest.java | 4 +- .../records/extent/MetadataExtentApiTest.java | 3 +- .../formatters/AbstractFormatterTest.java | 3 +- .../AlternateLogoForPdfExportTest.java | 2 +- .../FormatterApiIntegrationTest.java | 2 +- .../records/formatters/FormatterApiTest.java | 2 +- .../api/selections/UserSelectionsApiTest.java | 8 +- .../BatchOpsMetadatReindexerTest.java | 10 +- 69 files changed, 553 insertions(+), 373 deletions(-) create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index 4605016efd2..00e6a930e7d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -55,6 +55,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.repository.UserGroupRepository; import org.jdom.Element; import org.slf4j.Logger; @@ -164,8 +165,8 @@ public void indexMetadata(final List metadataIds) throws Exception { } @Deprecated - public void indexMetadata(final String metadataId, boolean forceRefreshReaders) throws Exception { - metadataIndexer.indexMetadata(metadataId, forceRefreshReaders, IndexingMode.full); + public void indexMetadata(final String metadataId, IIndexSubmittor indexSubmittor) throws Exception { + metadataIndexer.indexMetadata(metadataId, indexSubmittor, IndexingMode.full); } @Deprecated @@ -361,10 +362,10 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Deprecated public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, boolean forceRefreshReaders) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) throws Exception { return metadataManager.insertMetadata(context, newMetadata, metadataXml, indexingMode, updateFixedInfo, updateDatestamp, - fullRightsForGroup, forceRefreshReaders); + fullRightsForGroup, indexSubmittor); } @Deprecated diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index ac7db5535f3..663b5e50105 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -29,6 +29,7 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.User; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.utils.Log; import org.springframework.transaction.TransactionStatus; @@ -77,7 +78,7 @@ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metad } public void run() { - try { + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { _context.setAsThreadLocal(); while (_transactionStatus != null && !_transactionStatus.isCompleted()) { try { @@ -107,7 +108,7 @@ public void run() { } try { - dataManager.indexMetadata(metadataId.toString(), false); + dataManager.indexMetadata(metadataId.toString(), batchingIndexSubmittor); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Error indexing metadata '" + metadataId + "': " + e.getMessage() + "\n" + Util.getStackTrace(e)); @@ -116,7 +117,6 @@ public void run() { if (_user != null && _context.getUserSession().getUserId() == null) { _context.getUserSession().loginAs(_user); } - searchManager.forceIndexChanges(); } finally { _batchIndex.remove(this); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java index f7ef10c9c7c..09d9c76e5a6 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java @@ -28,6 +28,7 @@ import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.jdom.Element; import org.springframework.data.jpa.domain.Specification; @@ -92,7 +93,7 @@ public interface IMetadataIndexer { */ void indexMetadata(List metadataIds) throws Exception; - void indexMetadata(String metadataId, boolean forceRefreshReaders, IndexingMode indexingMode) throws Exception; + void indexMetadata(String metadataId, IIndexSubmittor indexSubmittor, IndexingMode indexingMode) throws Exception; void indexMetadataPrivileges(String uuid, int id) throws Exception; diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java index 83c4b8d05d9..f205284d013 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java @@ -33,6 +33,7 @@ import org.fao.geonet.kernel.EditLib; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.repository.BatchUpdateQuery; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -147,12 +148,12 @@ String insertMetadata(ServiceContext context, String schema, Element metadataXml * @param updateFixedInfo * @param updateDatestamp * @param fullRightsForGroup - * @param forceRefreshReaders + * @param indexSubmittor * @return * @throws Exception */ AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, boolean forceRefreshReaders) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) throws Exception; /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 9b39ce6118b..37162e0b5da 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -49,6 +49,8 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexFields; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -300,14 +302,16 @@ public boolean isIndexing() { @Override public void indexMetadata(final List metadataIds) throws Exception { - for (String metadataId : metadataIds) { - indexMetadata(metadataId, true, IndexingMode.full); + try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor()) { + for (String metadataId : metadataIds) { + indexMetadata(metadataId, indexSubmittor, IndexingMode.full); + } } } @Override public void indexMetadata(final String metadataId, - final boolean forceRefreshReaders, + final IIndexSubmittor indexSubmittor, final IndexingMode indexingMode) throws Exception { AbstractMetadata fullMd; @@ -388,7 +392,7 @@ public void indexMetadata(final String metadataId, "error", Map.of("record", metadataId, "schema", schema))); searchManager.index(null, md, indexKey, fields, metadataType, - forceRefreshReaders, indexingMode); + indexSubmittor, indexingMode); Log.error(Geonet.DATA_MANAGER, String.format( "Record %s / Schema '%s' is not registered in this catalog. Install it or remove those records. Record is indexed indexing error flag.", metadataId, schema)); @@ -533,7 +537,7 @@ public void indexMetadata(final String metadataId, } searchManager.index(schemaManager.getSchemaDir(schema), md, indexKey, fields, metadataType, - forceRefreshReaders, indexingMode); + indexSubmittor, indexingMode); } } catch (Exception x) { Log.error(Geonet.DATA_MANAGER, "The metadata document index with id=" + metadataId diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 7464a267735..f26a7479277 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -50,6 +50,8 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.kernel.setting.SettingInfo; @@ -464,7 +466,7 @@ public boolean apply(@Nullable MetadataCategory input) { newMetadata.getMetadataCategories().addAll(filteredCategories); int finalId = insertMetadata(context, newMetadata, xml, IndexingMode.full, true, UpdateDatestamp.YES, - fullRightsForGroup, true).getId(); + fullRightsForGroup, DirectIndexSubmittor.INSTANCE).getId(); return String.valueOf(finalId); } @@ -569,7 +571,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta boolean fullRightsForGroup = false; int finalId = insertMetadata(context, newMetadata, metadataXml, indexingMode, ufo, UpdateDatestamp.NO, - fullRightsForGroup, true).getId(); + fullRightsForGroup, DirectIndexSubmittor.INSTANCE).getId(); return String.valueOf(finalId); } @@ -577,7 +579,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Override public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, boolean updateFixedInfo, UpdateDatestamp updateDatestamp, - boolean fullRightsForGroup, boolean forceRefreshReaders) throws Exception { + boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) throws Exception { final String schema = newMetadata.getDataInfo().getSchemaId(); // Check if the schema is allowed by settings @@ -616,7 +618,7 @@ public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata metadataOperations.copyDefaultPrivForGroup(context, stringId, groupId, fullRightsForGroup); if (indexingMode != IndexingMode.none) { - metadataIndexer.indexMetadata(stringId, forceRefreshReaders, indexingMode); + metadataIndexer.indexMetadata(stringId, indexSubmittor, indexingMode); } return savedMetadata; @@ -789,7 +791,7 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) { getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo)); } - metadataIndexer.indexMetadata(metadataId, true, indexingMode); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, indexingMode); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java index cf5f7fc972d..1011cc3ffc8 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java @@ -38,6 +38,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataStatus; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataStatusRepository; @@ -155,7 +156,7 @@ public String getCurrentStatus(int metadataId) throws Exception { public MetadataStatus setStatus(ServiceContext context, int id, int status, ISODate changeDate, String changeMessage) throws Exception { MetadataStatus statusObject = setStatusExt(context, id, status, changeDate, changeMessage); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); return statusObject; } @@ -163,7 +164,7 @@ public MetadataStatus setStatus(ServiceContext context, int id, int status, ISOD public MetadataStatus setStatusExt(MetadataStatus metatatStatus, boolean updateIndex) throws Exception { metadataStatusRepository.save(metatatStatus); if (updateIndex) { - metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); } return metatatStatus; } @@ -250,7 +251,7 @@ public void changeCurrentStatus(Integer userId, Integer metadataId, Integer newS metatatStatus.setTitles(metadataUtils.extractTitles(Integer.toString(metadataId))); metadataStatusRepository.save(metatatStatus); - metadataIndexer.indexMetadata(metadataId + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); } // Utility to verify workflow status transitions diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java index ad4dde213b4..2e941b07505 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java @@ -43,6 +43,7 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.IndexingList; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.lib.Lib; @@ -512,7 +513,7 @@ public String getNewVersion(String id) { @Override public void setTemplate(final int id, final MetadataType type, final String title) throws Exception { setTemplateExt(id, type); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } @Override @@ -526,7 +527,7 @@ public void setTemplateExt(final int id, final MetadataType metadataType) throws @Override public void setHarvested(int id, String harvestUuid) throws Exception { setHarvestedExt(id, harvestUuid); - metadataIndexer.indexMetadata(Integer.toString(id), true, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } /** @@ -767,7 +768,7 @@ private void transformMd(ServiceContext context, String metadataId, Element md, getXmlSerializer().update(metadataId, md, changeDate, true, uuid, context); if (indexAfterChange) { - metadataIndexer.indexMetadata(metadataId, true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, IndexingMode.full); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index c5cc81ad1da..4c8cd76a1b6 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -41,6 +41,7 @@ import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; @@ -521,7 +522,7 @@ protected String createDraft(ServiceContext context, String templateId, String g try { newMetadata = (MetadataDraft) metadataManager.insertMetadata(context, newMetadata, xml, IndexingMode.full, true, - UpdateDatestamp.YES, false, true); + UpdateDatestamp.YES, false, DirectIndexSubmittor.INSTANCE); Integer finalId = newMetadata.getId(); diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java index 1200ff5e913..3f0e94e1cff 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java @@ -45,6 +45,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -444,7 +445,7 @@ public void apply(@Nonnull final Metadata metadata) { } } - dm.indexMetadata(metadataIdMap.get(index), true); + dm.indexMetadata(metadataIdMap.get(index), DirectIndexSubmittor.INSTANCE); } // -------------------------------------------------------------------- diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 169d5902075..50f23bce2ee 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -53,6 +53,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -88,8 +90,6 @@ public class EsSearchManager implements ISearchManager { public static final String FIELDNAME = "name"; public static final String FIELDSTRING = "string"; - private static final int COMMIT_INTERVAL = 200; - public static final Map RELATED_INDEX_FIELDS; public static final Set FIELDLIST_CORE; public static final Set FIELDLIST_RELATED; @@ -176,7 +176,6 @@ public void setIndexType(String indexType) { @Autowired private OverviewIndexFieldUpdater overviewFieldUpdater; - private final Map listOfDocumentsToIndex = Collections.synchronizedMap(new HashMap<>()); private Map indexList; private Path getXSLTForIndexing(Path schemaDir, MetadataType metadataType) { @@ -426,7 +425,7 @@ public void updateFieldAsynch(String id, String field, Object value) { public void index(Path schemaDir, Element metadata, String id, Multimap dbFields, MetadataType metadataType, - boolean forceRefreshReaders, + IIndexSubmittor indexSubmittor, IndexingMode indexingMode) throws Exception { Element docs = new Element("doc"); @@ -452,38 +451,17 @@ public void index(Path schemaDir, Element metadata, String id, String jsonDocument = mapper.writeValueAsString(doc); - listOfDocumentsToIndex.put(id, jsonDocument); - if (forceRefreshReaders || listOfDocumentsToIndex.size() >= COMMIT_INTERVAL) { - sendDocumentsToIndex(); - } + indexSubmittor.submitToIndex(id, jsonDocument, this); } - private void sendDocumentsToIndex() { - Map documents; - synchronized (listOfDocumentsToIndex) { - if (listOfDocumentsToIndex.isEmpty()) return; - documents = new HashMap<>(listOfDocumentsToIndex); - listOfDocumentsToIndex.clear(); + public void handleIndexResponse(BulkResponse bulkResponse, Map documents) throws IOException { + try { + checkIndexResponse(bulkResponse, documents); + } finally { + for (String uuid : documents.keySet()) { + overviewFieldUpdater.process(uuid); + } } - - BulkRequest bulkRequest = client.buildBulkRequest(defaultIndex, documents); - client.getAsyncClient().bulk(bulkRequest) - .thenAcceptAsync(bulkItemResponses -> { - try { - checkIndexResponse(bulkItemResponses, documents); - } catch (IOException e) { - throw new UncheckedIOException(e); - } finally { - documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid)); - } - } - ) - .exceptionally(e -> { - LOGGER.error( - "An error occurred while indexing {} documents in current indexing list.", - documents.size(), e); - return null; - }); } private void checkIndexResponse(BulkResponse bulkItemResponses, @@ -726,7 +704,7 @@ private String parseBoolean(String value) { @Override public void forceIndexChanges() { - sendDocumentsToIndex(); + // TODO remove - no longer needed due to new indexing behaviour } @Override @@ -739,44 +717,43 @@ public boolean rebuildIndex(ServiceContext context, clearIndex(); } - if (StringUtils.isNotBlank(bucket)) { - ArrayList listOfIdsToIndex = new ArrayList<>(); - UserSession session = context.getUserSession(); - SelectionManager sm = SelectionManager.getManager(session); - - synchronized (sm.getSelection(bucket)) { - for (Iterator iter = sm.getSelection(bucket).iterator(); - iter.hasNext(); ) { - String uuid = iter.next(); - for (AbstractMetadata metadata : metadataRepository.findAllByUuid(uuid)) { - String indexKey = uuid; - if (metadata instanceof MetadataDraft) { - indexKey += "-draft"; + try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor()) { + if (StringUtils.isNotBlank(bucket)) { + ArrayList listOfIdsToIndex = new ArrayList<>(); + UserSession session = context.getUserSession(); + SelectionManager sm = SelectionManager.getManager(session); + + synchronized (sm.getSelection(bucket)) { + for (String uuid : sm.getSelection(bucket)) { + for (AbstractMetadata metadata : metadataRepository.findAllByUuid(uuid)) { + String indexKey = uuid; + if (metadata instanceof MetadataDraft) { + indexKey += "-draft"; + } + + listOfIdsToIndex.add(indexKey); } - listOfIdsToIndex.add(indexKey); - } - - if (!metadataRepository.existsMetadataUuid(uuid)) { - LOGGER.warn("Selection contains uuid '{}' not found in database", uuid); + if (!metadataRepository.existsMetadataUuid(uuid)) { + LOGGER.warn("Selection contains uuid '{}' not found in database", uuid); + } } } - } - for (String id : listOfIdsToIndex) { - metadataIndexer.indexMetadata(id, false, IndexingMode.full); - } - } else { - final Specification metadataSpec = - Specification.where((Specification) MetadataSpecs.isType(MetadataType.METADATA)) - .or((Specification) MetadataSpecs.isType(MetadataType.TEMPLATE)); - final List metadataIds = metadataRepository.findAllIdsBy( - Specification.where(metadataSpec) - ); - for (Integer id : metadataIds) { - metadataIndexer.indexMetadata(id + "", false, IndexingMode.full); + for (String id : listOfIdsToIndex) { + metadataIndexer.indexMetadata(id, indexSubmittor, IndexingMode.full); + } + } else { + final Specification metadataSpec = + Specification.where((Specification) MetadataSpecs.isType(MetadataType.METADATA)) + .or((Specification) MetadataSpecs.isType(MetadataType.TEMPLATE)); + final List metadataIds = metadataRepository.findAllIdsBy( + Specification.where(metadataSpec) + ); + for (Integer id : metadataIds) { + metadataIndexer.indexMetadata(id + "", indexSubmittor, IndexingMode.full); + } } } - sendDocumentsToIndex(); return true; } @@ -931,7 +908,8 @@ public static String analyzeField(String analyzer, } public boolean isIndexing() { - return !listOfDocumentsToIndex.isEmpty(); + // TODO this needs a new metric + return false; } public boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index f7d4959953b..67e40b64e8f 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -28,6 +28,7 @@ import jeeves.server.context.ServiceContext; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.MetadataType; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.jdom.Element; import java.io.IOException; @@ -47,11 +48,10 @@ public interface ISearchManager { /** * Indexes a metadata record. * - * @param forceRefreshReaders if true then block all searches until they can obtain a up-to-date - * reader + * @param indexSubmittor The submittor to use */ void index(Path schemaDir, Element metadata, String id, Multimap moreFields, - MetadataType metadataType, boolean forceRefreshReaders, IndexingMode indexingMode) + MetadataType metadataType, IIndexSubmittor indexSubmittor, IndexingMode indexingMode) throws Exception; /** diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index 22ad1941980..c782cefb392 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -51,6 +51,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.MetadataIndexerProcessor; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.util.ThreadUtils; import org.fao.geonet.utils.Log; import org.springframework.jmx.export.MBeanExporter; @@ -208,20 +209,21 @@ public void run() { "Indexing range [%d-%d]/%d by threads %s.", beginIndex, beginIndex + count, ids.length, Thread.currentThread().getId())); long start = System.currentTimeMillis(); - for (int i = beginIndex; i < beginIndex + count; i++) { - try { - dm.indexMetadata(ids[i] + "", false); - processed.incrementAndGet(); - } catch (Exception e) { - inError.incrementAndGet(); + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { + for (int i = beginIndex; i < beginIndex + count; i++) { + try { + dm.indexMetadata(ids[i] + "", batchingIndexSubmittor); + processed.incrementAndGet(); + } catch (Exception e) { + inError.incrementAndGet(); + } } + Log.warning(Geonet.INDEX_ENGINE, String.format( + "Indexing range [%d-%d]/%d completed in %dms by threads %s.", + beginIndex, beginIndex + count, ids.length, + System.currentTimeMillis() - start, + Thread.currentThread().getId())); } - Log.warning(Geonet.INDEX_ENGINE, String.format( - "Indexing range [%d-%d]/%d completed in %dms by threads %s.", - beginIndex, beginIndex + count, ids.length, - System.currentTimeMillis() - start, - Thread.currentThread().getId())); - ApplicationContextHolder.get().getBean(EsSearchManager.class).forceIndexChanges(); } } } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index dfb4eb305ec..1166a8e8108 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -30,6 +30,7 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.utils.Log; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -63,7 +64,11 @@ private void indexRecords() { ApplicationContextHolder.set(applicationContext); IndexingList list = applicationContext.getBean(IndexingList.class); Set metadataIdentifiers = list.getIdentifiers(); - if (metadataIdentifiers.size() > 0) { + if (metadataIdentifiers.isEmpty()) { + return; + } + + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) { Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " + metadataIdentifiers.toString() + "."); @@ -71,13 +76,12 @@ private void indexRecords() { for (Integer metadataIdentifier : metadataIdentifiers) { try { - _dataManager.indexMetadata(String.valueOf(metadataIdentifier), false); + _dataManager.indexMetadata(String.valueOf(metadataIdentifier), batchingIndexSubmittor); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Indexing task / An error happens indexing the metadata " + metadataIdentifier + ". Error: " + e.getMessage(), e); } } - this.applicationContext.getBean(EsSearchManager.class).forceIndexChanges(); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java new file mode 100644 index 00000000000..02c5d71eac8 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java @@ -0,0 +1,100 @@ +package org.fao.geonet.kernel.search.submission; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import org.fao.geonet.constants.Geonet; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * An index submitter that batches documents into larger chunks and sends them asynchronously to the index. + * When closing, this submittor sends the remaining item to the index and waits until all elasticsearch requests have been received, + * so after closing there are no pending changes + */ +public class BatchingIndexSubmittor implements AutoCloseable, IIndexSubmittor { + private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.INDEX_ENGINE); + /** + * Maximum number of inflight bulk requests before waiting for the Elasticsearch + */ + private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; + private static final int COMMIT_INTERVAL = 200; + @SuppressWarnings("unchecked") + private final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; + private final Map listOfDocumentsToIndex = new HashMap<>(); + private int index; + private boolean closed = false; + private EsSearchManager searchManager; + + @Override + public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { + if (closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + this.searchManager = searchManager; + listOfDocumentsToIndex.put(id, jsonDocument); + if (listOfDocumentsToIndex.size() >= COMMIT_INTERVAL) { + Map toIndex = new HashMap<>(listOfDocumentsToIndex); + listOfDocumentsToIndex.clear(); + sendDocumentsToIndex(toIndex); + } + } + + @Override + public void close() { + if (closed) { + throw new IllegalStateException("Attempting to close a already closed " + this.getClass().getSimpleName()); + } + this.closed = true; + + // Send any remaining pending documents + if (!this.listOfDocumentsToIndex.isEmpty()) { + sendDocumentsToIndex(this.listOfDocumentsToIndex); + } + + // Wait for all remaining documents to be received + for (CompletableFuture inflightFuture : inflightFutures) { + if (inflightFuture != null) { + inflightFuture.join(); + } + } + } + + private void sendDocumentsToIndex(Map toIndex) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), listOfDocumentsToIndex); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + try { + searchManager.handleIndexResponse(bulkItemResponses, listOfDocumentsToIndex); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).exceptionally(e -> { + LOGGER.error( + "An error occurred while indexing {} documents in current indexing list.", + toIndex.size(), e); + return null; + }); + + // Request send, sort into queue + // First, see if the previous future already finished + CompletableFuture previousFuture = inflightFutures[index]; + if (previousFuture != null) { + if (!previousFuture.isDone()) { + // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer + LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); + previousFuture.join(); + } + } + inflightFutures[index] = currentIndexFuture; + index = (index + 1) % inflightFutures.length; + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java new file mode 100644 index 00000000000..e6500bb4e42 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java @@ -0,0 +1,30 @@ +package org.fao.geonet.kernel.search.submission; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * An index submittor that directly and synchronously transmits new documents to the index + */ +public class DirectIndexSubmittor implements IIndexSubmittor { + public static final DirectIndexSubmittor INSTANCE = new DirectIndexSubmittor(); + + private DirectIndexSubmittor() {} + + @Override + public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + Map documents = Collections.singletonMap(id, jsonDocument); + + BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), documents); + final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); + + searchManager.handleIndexResponse(bulkItemResponses, documents); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java new file mode 100644 index 00000000000..d65d65a7350 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java @@ -0,0 +1,10 @@ +package org.fao.geonet.kernel.search.submission; + +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; + +public interface IIndexSubmittor { + + void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException; +} diff --git a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java index 5bd12fcb26f..09627d01e43 100644 --- a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java @@ -49,6 +49,8 @@ import org.fao.geonet.kernel.mef.Importer; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.repository.AbstractSpringDataTest; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; @@ -350,7 +352,7 @@ public int importMetadataXML(ServiceContext context, String uuid, InputStream xm id, createDate, createDate, "" + groupId, metadataType); - dataManager.indexMetadata(id.get(0), true); + dataManager.indexMetadata(id.get(0), DirectIndexSubmittor.INSTANCE); return Integer.parseInt(id.get(0)); } @@ -386,11 +388,7 @@ public boolean resetLuceneIndex() { return true; } - protected AbstractMetadata injectMetadataInDbDoNotRefreshHeader(Element sampleMetadataXml, ServiceContext context) throws Exception { - return injectMetadataInDb(sampleMetadataXml, context, false); - } - - protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, ServiceContext context, boolean resfreshHeader) throws Exception { + protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, ServiceContext context) throws Exception { String uuid = UUID.randomUUID().toString(); String schema = schemaManager.autodetectSchema(sampleMetadataXml); Xml.selectElement(sampleMetadataXml, @@ -419,6 +417,6 @@ protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, Service .setHarvested(false); return metadataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.none, false, UpdateDatestamp.NO, - false, resfreshHeader); + false, DirectIndexSubmittor.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java index 2238444e734..e4bb80e5f75 100644 --- a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java +++ b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java @@ -7,6 +7,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.SourceRepository; import org.jdom.Element; import org.springframework.beans.factory.annotation.Autowired; @@ -65,6 +66,6 @@ protected AbstractMetadata insertTemplateResourceInDb(ServiceContext serviceCont false, NO, false, - true); + DirectIndexSubmittor.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index f81456e4f84..fa7b010b9b1 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -74,7 +74,7 @@ public void testDeleteMetadata() throws Exception { ServiceContext serviceContext = createContextAndLogAsAdmin(); long count = metadataRepository.count(); - int mdId = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), serviceContext).getId(); + int mdId = injectMetadataInDb(getSampleMetadataXml(), serviceContext).getId(); assertEquals(count + 1, metadataRepository.count()); diff --git a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java index 607ea1f58c6..f43915553b0 100644 --- a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java @@ -11,6 +11,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.LinkRepository; import org.fao.geonet.repository.LinkStatusRepository; @@ -301,7 +302,7 @@ private AbstractMetadata insertMetadataInDb(Element element) throws Exception { false, NO, false, - false); + DirectIndexSubmittor.INSTANCE); return dbInsertedMetadata; } diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java index f571bb968bc..4d7a445413d 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java @@ -50,6 +50,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Log; @@ -279,7 +280,7 @@ private boolean insertTransaction(Element xml, List documents, dataMan.setOperation(context, id, "" + ReservedGroup.all.getId(), ReservedOperation.dynamic); } - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); AbstractMetadata metadata = metadataUtils.findOne(id); ApplicationContext applicationContext = ApplicationContextHolder.get(); diff --git a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java index e53ddb3cd87..e51a2219344 100644 --- a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java +++ b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java @@ -36,6 +36,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepositoryTest; import org.fao.geonet.utils.Xml; import org.jdom.Content; @@ -390,7 +391,7 @@ private void addPhotographicMetadataToRepository(int ownerId) throws Exception { extras.put("_isTemplate", "n"); extras.put("_owner", "" + ownerId); _searchManager.index(schemaDir, metadata.getXmlData(false), "" + metadata.getId(), extras, - MetadataType.METADATA,false, IndexingMode.full); + MetadataType.METADATA, DirectIndexSubmittor.INSTANCE, IndexingMode.full); } private Element createUpdateTransaction(String property, Object newValue) { diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java index 101f6fd78ab..09ad3b5af6e 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java @@ -35,6 +35,7 @@ import org.fao.geonet.kernel.harvest.harvester.CategoryMapper; import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.Privileges; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataCategoryRepository; @@ -63,14 +64,16 @@ * * @author heikki doeleman */ -public abstract class BaseAligner

extends AbstractAligner

{ +public abstract class BaseAligner

extends AbstractAligner

implements AutoCloseable { - private static Logger LOGGER = LoggerFactory.getLogger(Geonet.HARVESTER); + private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.HARVESTER); public final AtomicBoolean cancelMonitor; + protected final BatchingIndexSubmittor batchingIndexSubmittor; protected BaseAligner(AtomicBoolean cancelMonitor) { this.cancelMonitor = cancelMonitor; + this.batchingIndexSubmittor = new BatchingIndexSubmittor(); } public void addCategories(AbstractMetadata metadata, Iterable categories, @@ -199,4 +202,8 @@ public Element translateMetadataContent(ServiceContext context, return md; } + @Override + public void close() throws Exception { + this.batchingIndexSubmittor.close(); + } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java index 8b6e7881742..5e847b8a718 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java @@ -42,6 +42,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -270,26 +272,25 @@ private void align(Map metadataList) throws Exception { continue; } - BaseAligner aligner = new BaseAligner(cancelMonitor) { - }; - aligner.setParams(params); - // - // add / update the metadata from this harvesting result - // - String id = dataMan.getMetadataId(uuid); - if (id == null) { - id = addMetadata(metadataElement, uuid, schema, localGroups, localCateg, aligner); - result.addedMetadata++; - } else { - updateMetadata(metadataElement, id, localGroups, localCateg, aligner); - result.updatedMetadata++; - } + try (BaseAligner aligner = new BaseAligner<>(cancelMonitor) {}) { + aligner.setParams(params); + // + // add / update the metadata from this harvesting result + // + String id = dataMan.getMetadataId(uuid); + if (id == null) { + id = addMetadata(metadataElement, uuid, schema, localGroups, localCateg, aligner); + result.addedMetadata++; + } else { + updateMetadata(metadataElement, id, localGroups, localCateg, aligner); + result.updatedMetadata++; + } + if (StringUtils.isNotEmpty(thumbnailContent)) { + loadMetadataThumbnail(thumbnailContent, id, uuid); + } - if (StringUtils.isNotEmpty(thumbnailContent)) { - loadMetadataThumbnail(thumbnailContent, id, uuid); + idsForHarvestingResult.add(Integer.valueOf(id)); } - - idsForHarvestingResult.add(Integer.valueOf(id)); } } }catch(Throwable t) { @@ -320,7 +321,7 @@ private void align(Map metadataList) throws Exception { } } - private void updateMetadata(Element xml, String id, GroupMapper localGroups, final CategoryMapper localCateg, BaseAligner aligner) throws Exception { + private void updateMetadata(Element xml, String id, GroupMapper localGroups, final CategoryMapper localCateg, BaseAligner aligner) throws Exception { log.info("Updating metadata with id: " + id); // @@ -352,14 +353,14 @@ private void updateMetadata(Element xml, String id, GroupMapper localGroups, fin metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } /** * Inserts a metadata into the database. Lucene index is updated after insertion. */ private String addMetadata(Element xml, String uuid, String schema, GroupMapper localGroups, final CategoryMapper localCateg, - BaseAligner aligner) throws Exception { + BaseAligner aligner) throws Exception { log.info(" - Adding metadata with remote uuid: " + uuid); // @@ -396,13 +397,13 @@ private String addMetadata(Element xml, String uuid, String schema, GroupMapper aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); String id = String.valueOf(metadata.getId()); aligner.addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 5097d9a600c..50388cc5044 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -365,13 +365,13 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); result.addedMetadata++; } @@ -435,7 +435,7 @@ private void updateMetadata(RecordInfo ri, String id, boolean force) throws Exce } else { log.debug(" - Updating local metadata for uuid:" + ri.uuid); if (updatingLocalMetadata(ri, id, force)) { - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); result.updatedMetadata++; } } @@ -541,7 +541,7 @@ private Element retrieveMetadata(String uuid) { params.getValidate().validate(dataMan, context, response, groupIdVal); } catch (Exception e) { - log.info("Ignoring invalid metadata with uuid " + uuid); + log.info("Ignoring invalid metadata with uuid " + uuid, e); result.doesNotValidate++; return null; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java index fa088a5a4cb..e42b5e8dcb1 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Harvester.java @@ -110,9 +110,8 @@ public HarvestResult harvest(Logger log) throws Exception { boolean error = false; HarvestResult result = new HarvestResult(); - Set uuids = new HashSet(); - try { - Aligner aligner = new Aligner(cancelMonitor, context, server, params, log); + Set uuids = new HashSet<>(); + try (Aligner aligner = new Aligner(cancelMonitor, context, server, params, log)) { searchAndAlign(server, uuids, aligner, errors); result = aligner.cleanupRemovedRecords(uuids); } catch (Exception t) { diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java index cea522c6567..73e1d653d4a 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java @@ -403,14 +403,14 @@ private void createSubtemplate(String schema, Element md, String uuid, String ti addCategories(metadata, params.categories, localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); // Note: we use fragmentAllPrivs here because subtemplates need to be // visible/accessible to all addPrivileges(id, fragmentAllPrivs, localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmittor); metadataManager.flush(); @@ -594,7 +594,7 @@ private void update(String id, Element template, String title, boolean isSubtemp } dataMan.setHarvestedExt(iId, params.uuid, Optional.of(harvestUri)); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmittor); metadataManager.flush(); } @@ -634,7 +634,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } metadata.getCategories().add(metadataCategory); } - metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); @@ -643,7 +643,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } addPrivileges(id, params.privileges, localGroups, context); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmittor); if (log.isDebugEnabled()) { log.debug(" - Commit " + id); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index 0a3c5b7bd9d..153f6bdec24 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -191,13 +191,13 @@ private void addMetadata(RecordInfo ri) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, batchingIndexSubmittor); result.addedMetadata++; } @@ -241,7 +241,7 @@ private void updateMetadata(RecordInfo ri, String id) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, true); metadataManager.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, batchingIndexSubmittor); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java index 8d519a1b2d5..3f457979600 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Harvester.java @@ -151,9 +151,9 @@ public HarvestResult harvest(Logger log) throws Exception { //--- align local node - Aligner aligner = new Aligner(cancelMonitor, log, context, params); - - return aligner.align(records, errors); + try (Aligner aligner = new Aligner(cancelMonitor, log, context, params)) { + return aligner.align(records, errors); + } } /** diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java index 4f6418d5ddf..7fb63ce035e 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java @@ -255,8 +255,6 @@ public HarvestResult align(SortedSet records, List err } } - dataMan.forceIndexChanges(); - log.info("End of alignment for : " + params.getName()); return result; @@ -517,7 +515,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, this.batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); @@ -546,7 +544,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca } context.getBean(IMetadataManager.class).save(metadata); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, this.batchingIndexSubmittor); result.addedMetadata++; return id; @@ -843,7 +841,7 @@ private void updateMetadata(RecordInfo ri, String id, Element md, metadataManager.save(metadata); // dataMan.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, this.batchingIndexSubmittor); } private void handleFile(String id, String file, MetadataResourceVisibility visibility, String changeDate, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java index 96ac8c5660b..ec5a74f5157 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Harvester.java @@ -226,8 +226,9 @@ public HarvestResult harvest(Logger log) throws Exception { HarvestResult result = new HarvestResult(); if (!error) { try { - Aligner aligner = new Aligner(cancelMonitor, log, context, req, params, remoteInfo); - result = aligner.align(records, errors); + try (Aligner aligner = new Aligner(cancelMonitor, log, context, req, params, remoteInfo)) { + result = aligner.align(records, errors); + } Map sources = buildSources(remoteInfo); updateSources(records, sources); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java index 93f48ff9a48..c1af6cdbcae 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java @@ -42,6 +42,7 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.repository.specification.MetadataCategorySpecs; import org.fao.geonet.utils.XmlRequest; @@ -157,7 +158,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { //--- maybe the metadata was unretrievable if (id != null) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } } } @@ -212,7 +213,7 @@ private String addMetadata(Element info) throws Exception { List categories = info.getChildren("category"); addCategories(metadata, categories); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); String id = String.valueOf(metadata.getId()); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java index 640ddbee67b..1b723a12d06 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.specification.MetadataSpecs; import org.fao.geonet.utils.IO; @@ -90,48 +91,50 @@ protected LocalFilesystemParams createParams() { */ private HarvestResult align(Path root) throws Exception { log.debug("Start of alignment for : " + params.getName()); - final LocalFsHarvesterFileVisitor visitor = new LocalFsHarvesterFileVisitor(cancelMonitor, context, params, this); - if (params.recurse) { - Files.walkFileTree(root, visitor); - } else { - try (DirectoryStream paths = Files.newDirectoryStream(root)) { - for (Path path : paths) { - if (path != null && Files.isRegularFile(path)) { - visitor.visitFile(path, Files.readAttributes(path, BasicFileAttributes.class)); + try (final LocalFsHarvesterFileVisitor visitor = new LocalFsHarvesterFileVisitor(cancelMonitor, context, params, this)) { + if (params.recurse) { + Files.walkFileTree(root, visitor); + } else { + try (DirectoryStream paths = Files.newDirectoryStream(root)) { + for (Path path : paths) { + if (path != null && Files.isRegularFile(path)) { + visitor.visitFile(path, Files.readAttributes(path, BasicFileAttributes.class)); + } } } } - } - result = visitor.getResult(); - log.debug(String.format("Scan directory is done. %d files analyzed.", result.totalMetadata)); - Set idsForHarvestingResult = visitor.getListOfRecords(); - Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); - - if (!params.nodelete) { - log.debug("Starting to delete locally existing metadata " + - "from the same source if they " + - " were not in this harvesting result..."); - List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - for (Integer existingId : existingMetadata) { - - if (cancelMonitor.get()) { - return this.result; - } - if (!idsResultHs.contains(existingId)) { - log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString()); - result.locallyRemoved++; + result = visitor.getResult(); + log.debug(String.format("Scan directory is done. %d files analyzed.", result.totalMetadata)); + Set idsForHarvestingResult = visitor.getListOfRecords(); + Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); + + if (!params.nodelete) { + log.debug("Starting to delete locally existing metadata " + + "from the same source if they " + + " were not in this harvesting result..."); + List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); + for (Integer existingId : existingMetadata) { + + if (cancelMonitor.get()) { + return this.result; + } + if (!idsResultHs.contains(existingId)) { + log.debug(" Removing: " + existingId); + metadataManager.deleteMetadata(context, existingId.toString()); + result.locallyRemoved++; + } } } - } - log.debug("Starting indexing in batch thread pool..."); + log.debug("Starting indexing in batch thread pool..."); + + List listOfRecordsToIndex = Lists.newArrayList(visitor.getListOfRecordsToIndex()); - List listOfRecordsToIndex = Lists.newArrayList(visitor.getListOfRecordsToIndex()); - log.debug(String.format( - "Starting indexing in batch thread pool of %d updated records ...", - listOfRecordsToIndex.size())); - dataMan.batchIndexInThreadPool(context, listOfRecordsToIndex); + log.debug(String.format( + "Starting indexing in batch thread pool of %d updated records ...", + listOfRecordsToIndex.size())); + dataMan.batchIndexInThreadPool(context, listOfRecordsToIndex); + } log.debug("End of alignment for : " + params.getName()); return result; @@ -170,7 +173,7 @@ void updateMetadata(Element xml, final String id, GroupMapper localGroups, final metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } String addMetadata(Element xml, String uuid, String schema, GroupMapper localGroups, final CategoryMapper localCateg, @@ -218,7 +221,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); String id = String.valueOf(metadata.getId()); @@ -227,7 +230,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro metadataManager.flush(); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java index 791f9a17cda..172e07eda9f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFsHarvesterFileVisitor.java @@ -72,7 +72,7 @@ /** * @author Jesse on 11/6/2014. */ -class LocalFsHarvesterFileVisitor extends SimpleFileVisitor { +class LocalFsHarvesterFileVisitor extends SimpleFileVisitor implements AutoCloseable { private final LocalFilesystemParams params; private final DataManager dataMan; @@ -503,4 +503,9 @@ public Set getListOfRecords() { public Set getListOfRecordsToIndex() { return listOfRecordsToIndex; } + + @Override + public void close() throws Exception { + this.aligner.close(); + } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java index 79bc1fb174b..9871f5aeecc 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java @@ -431,7 +431,7 @@ private void addMetadata(XmlRequest t, RecordInfo ri, String processName, Map addMetadata(Element capa) throws Exception { if (!dataMan.existsMetadataUuid(uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(uuid); @@ -834,7 +834,7 @@ private WxSLayerRegistry addLayerMetadata(Element layer, Element capa) throws JD } if (!dataMan.existsMetadataUuid(reg.uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(reg.uuid); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java index 39bcaee8ea1..148249b4a7c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/OgcWxSHarvester.java @@ -74,8 +74,9 @@ protected void storeNodeExtra(OgcWxSParams params, String path, //--------------------------------------------------------------------------- public void doHarvest(Logger log) throws Exception { - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java index 4bb78e3dee8..a7ca4eb1874 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java @@ -249,13 +249,13 @@ private void addMetadata(Map.Entry recordInfo, String overrideU addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); result.addedMetadata++; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java index 254fac91f84..546b0658bb3 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Harvester.java @@ -104,105 +104,106 @@ public HarvestResult harvest(Logger log) throws Exception { String[] urlList = params.url.split("\n"); boolean error = false; - Aligner aligner = new Aligner(cancelMonitor, context, params, log); - Set listOfUuids = new HashSet<>(); - - for (String url : urlList) { - log.debug("Loading URL: " + url); - String content = retrieveUrl(url); - if (cancelMonitor.get()) { - return new HarvestResult(); - } - log.debug("Response is: " + content); + try (Aligner aligner = new Aligner(cancelMonitor, context, params, log)) { + Set listOfUuids = new HashSet<>(); + + for (String url : urlList) { + log.debug("Loading URL: " + url); + String content = retrieveUrl(url); + if (cancelMonitor.get()) { + return new HarvestResult(); + } + log.debug("Response is: " + content); - int numberOfRecordsToHarvest = -1; + int numberOfRecordsToHarvest = -1; - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonObj = null; - Element xmlObj = null; - SimpleUrlResourceType type; + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonObj = null; + Element xmlObj = null; + SimpleUrlResourceType type; - if (isRDFLike(content)) type = SimpleUrlResourceType.RDFXML; - else if (isXMLLike(content)) type = SimpleUrlResourceType.XML; - else type = SimpleUrlResourceType.JSON; + if (isRDFLike(content)) type = SimpleUrlResourceType.RDFXML; + else if (isXMLLike(content)) type = SimpleUrlResourceType.XML; + else type = SimpleUrlResourceType.JSON; - if (type == SimpleUrlResourceType.XML - || type == SimpleUrlResourceType.RDFXML) { - xmlObj = Xml.loadString(content, false); - } else { - jsonObj = objectMapper.readTree(content); - } - - // TODO: Add page support for Hydra in RDFXML feeds ? - if (StringUtils.isNotEmpty(params.numberOfRecordPath)) { - try { - if (type == SimpleUrlResourceType.XML) { - Object element = Xml.selectSingle(xmlObj, params.numberOfRecordPath, xmlObj.getAdditionalNamespaces()); - if (element != null) { - String s = getXmlElementTextValue(element); - numberOfRecordsToHarvest = Integer.parseInt(s); - } - } else if (type == SimpleUrlResourceType.JSON) { - numberOfRecordsToHarvest = jsonObj.at(params.numberOfRecordPath).asInt(); - } - log.debug("Number of records to harvest: " + numberOfRecordsToHarvest); - } catch (Exception e) { - errors.add(new HarvestError(context, e)); - log.error(String.format("Failed to extract total in response at path %s. Error is: %s", - params.numberOfRecordPath, e.getMessage())); + if (type == SimpleUrlResourceType.XML + || type == SimpleUrlResourceType.RDFXML) { + xmlObj = Xml.loadString(content, false); + } else { + jsonObj = objectMapper.readTree(content); } - } - try { - List listOfUrlForPages = buildListOfUrl(params, numberOfRecordsToHarvest); - for (int i = 0; i < listOfUrlForPages.size(); i++) { - if (i != 0) { - content = retrieveUrl(listOfUrlForPages.get(i)); + + // TODO: Add page support for Hydra in RDFXML feeds ? + if (StringUtils.isNotEmpty(params.numberOfRecordPath)) { + try { if (type == SimpleUrlResourceType.XML) { - xmlObj = Xml.loadString(content, false); - } else { - jsonObj = objectMapper.readTree(content); + Object element = Xml.selectSingle(xmlObj, params.numberOfRecordPath, xmlObj.getAdditionalNamespaces()); + if (element != null) { + String s = getXmlElementTextValue(element); + numberOfRecordsToHarvest = Integer.parseInt(s); + } + } else if (type == SimpleUrlResourceType.JSON) { + numberOfRecordsToHarvest = jsonObj.at(params.numberOfRecordPath).asInt(); } + log.debug("Number of records to harvest: " + numberOfRecordsToHarvest); + } catch (Exception e) { + errors.add(new HarvestError(context, e)); + log.error(String.format("Failed to extract total in response at path %s. Error is: %s", + params.numberOfRecordPath, e.getMessage())); } - if (StringUtils.isNotEmpty(params.loopElement) - || type == SimpleUrlResourceType.RDFXML) { - Map uuids = new HashMap<>(); - try { + } + try { + List listOfUrlForPages = buildListOfUrl(params, numberOfRecordsToHarvest); + for (int i = 0; i < listOfUrlForPages.size(); i++) { + if (i != 0) { + content = retrieveUrl(listOfUrlForPages.get(i)); if (type == SimpleUrlResourceType.XML) { - collectRecordsFromXml(xmlObj, uuids, aligner); - } else if (type == SimpleUrlResourceType.RDFXML) { - collectRecordsFromRdf(xmlObj, uuids, aligner); - } else if (type == SimpleUrlResourceType.JSON) { - collectRecordsFromJson(jsonObj, uuids, aligner); + xmlObj = Xml.loadString(content, false); + } else { + jsonObj = objectMapper.readTree(content); + } + } + if (StringUtils.isNotEmpty(params.loopElement) + || type == SimpleUrlResourceType.RDFXML) { + Map uuids = new HashMap<>(); + try { + if (type == SimpleUrlResourceType.XML) { + collectRecordsFromXml(xmlObj, uuids, aligner); + } else if (type == SimpleUrlResourceType.RDFXML) { + collectRecordsFromRdf(xmlObj, uuids, aligner); + } else if (type == SimpleUrlResourceType.JSON) { + collectRecordsFromJson(jsonObj, uuids, aligner); + } + aligner.align(uuids, errors); + listOfUuids.addAll(uuids.keySet()); + } catch (Exception e) { + errors.add(new HarvestError(this.context, e)); + log.error(String.format("Failed to collect record in response at path %s. Error is: %s", + params.loopElement, e.getMessage())); } - aligner.align(uuids, errors); - listOfUuids.addAll(uuids.keySet()); - } catch (Exception e) { - errors.add(new HarvestError(this.context, e)); - log.error(String.format("Failed to collect record in response at path %s. Error is: %s", - params.loopElement, e.getMessage())); } } + } catch (Exception t) { + error = true; + log.error("Unknown error trying to harvest"); + log.error(t.getMessage()); + log.error(t); + errors.add(new HarvestError(context, t)); + } catch (Throwable t) { + error = true; + log.fatal("Something unknown and terrible happened while harvesting"); + log.fatal(t.getMessage()); + errors.add(new HarvestError(context, t)); } - } catch (Exception t) { - error = true; - log.error("Unknown error trying to harvest"); - log.error(t.getMessage()); - log.error(t); - errors.add(new HarvestError(context, t)); - } catch (Throwable t) { - error = true; - log.fatal("Something unknown and terrible happened while harvesting"); - log.fatal(t.getMessage()); - errors.add(new HarvestError(context, t)); - } - log.info("Total records processed in all searches :" + listOfUuids.size()); - if (error) { - log.warning("Due to previous errors the align process has not been called"); + log.info("Total records processed in all searches :" + listOfUuids.size()); + if (error) { + log.warning("Due to previous errors the align process has not been called"); + } } + aligner.cleanupRemovedRecords(listOfUuids); + return aligner.getResult(); } - aligner.cleanupRemovedRecords(listOfUuids); - return aligner.getResult(); } private void collectRecordsFromJson(JsonNode jsonObj, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java index 80304603e62..ea60589cbe9 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java @@ -56,6 +56,7 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UriMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.util.Sha1Encoder; @@ -572,13 +573,13 @@ private void saveMetadata(Element md, String uuid, String uri, boolean isService addCategories(metadata, params.getCategories(), localCateg, context, null, false); } - metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - mdIndexer.indexMetadata(id, true, IndexingMode.full); + mdIndexer.indexMetadata(id, DirectIndexSubmittor.INSTANCE, IndexingMode.full); mdManager.flush(); } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java index 32ca2ff5296..2b5c7e7324c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/ThreddsHarvester.java @@ -72,7 +72,8 @@ protected void storeNodeExtra(ThreddsParams params, String path, //--------------------------------------------------------------------------- public void doHarvest(Logger log) throws Exception { - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java index cf8717e5213..0c156465682 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java @@ -291,7 +291,7 @@ private void addMetadata(RemoteFile rf) throws Exception { if (StringUtils.isNotEmpty(params.xslfilter)) { md = HarvesterUtil.processMetadata(dataMan.getSchema(schema), md, processName, processParams); - + schema = dataMan.autodetectSchema(md); } @@ -340,14 +340,14 @@ private void addMetadata(RemoteFile rf) throws Exception { } catch (NumberFormatException e) { } - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, false); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); metadataManager.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, batchingIndexSubmittor); result.addedMetadata++; } @@ -522,7 +522,7 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force) dataMan.flush(); - dataMan.indexMetadata(recordInfo.id, true); + dataMan.indexMetadata(recordInfo.id, batchingIndexSubmittor); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java index e745a5b3311..4a87b8573a3 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/WebDavHarvester.java @@ -47,8 +47,9 @@ protected void storeNodeExtra(WebDavParams params, String path, String siteId, S public void doHarvest(Logger log) throws Exception { log.info("WebDav doHarvest start"); - Harvester h = new Harvester(cancelMonitor, log, context, params); - result = h.harvest(log); + try (Harvester h = new Harvester(cancelMonitor, log, context, params)) { + result = h.harvest(log); + } log.info("WebDav doHarvest end"); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java index a0d72818fac..30bfb785647 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java @@ -212,12 +212,12 @@ public HarvestResult harvest(Logger log) throws Exception { } //--- harvest metadata and subtemplates from fragments using generic fragment harvester - FragmentHarvester fragmentHarvester = new FragmentHarvester(cancelMonitor, log, context, getFragmentHarvesterParams()); - - if (params.streamFeatures) { - harvestFeatures(wfsQuery, fragmentHarvester); - } else { - harvestResponse(wfsQuery, fragmentHarvester); + try (FragmentHarvester fragmentHarvester = new FragmentHarvester(cancelMonitor, log, context, getFragmentHarvesterParams())) { + if (params.streamFeatures) { + harvestFeatures(wfsQuery, fragmentHarvester); + } else { + harvestResponse(wfsQuery, fragmentHarvester); + } } return result; diff --git a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java index 1387a2c8e87..5bf5092f77e 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java @@ -36,6 +36,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataRepository; @@ -111,7 +112,7 @@ public static Element process(ServiceContext context, String id, dataMan.updateMetadata(context, id, wellFormedXml, validate, ufo, language, new ISODate().toString(), updateDateStamp, index ? IndexingMode.full : IndexingMode.none); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } } } diff --git a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java index e8d52b5dc8b..cc842613b73 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java @@ -35,6 +35,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataValidationRepository; @@ -178,7 +179,7 @@ public void apply(@Nonnull AbstractMetadata entity) { dataMan.updateMetadata(context, id, processedMetadata, validate, ufo, language, new ISODate().toString(), updateDateStamp, IndexingMode.none); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); } } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index ad25403313c..2d00792fdce 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -80,6 +80,7 @@ import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataDraftRepository; @@ -1022,7 +1023,7 @@ private Pair loadRecord(MetadataType metadataType, Element xmlE metadataValidator.doValidate(metadata, context.getLanguage()); } - dataManager.indexMetadata(id.get(0), true); + dataManager.indexMetadata(id.get(0), DirectIndexSubmittor.INSTANCE); return Pair.read(Integer.valueOf(id.get(0)), uuid); } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java index fd1396387be..48b15b9a2c9 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java @@ -50,6 +50,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Log; import org.fao.geonet.utils.Xml; @@ -290,7 +291,7 @@ SimpleMetadataProcessingReport addTemplates( if (MetadataType.lookup(isTemplate) == MetadataType.TEMPLATE) { xml = dataManager.setUUID(schemaName, uuid, xml); } - dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, true); + dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); report.addMetadataInfos(metadata, String.format( "Template for schema '%s' with UUID '%s' added.", diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java index 5967992f740..22c50ad1f55 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java @@ -55,6 +55,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.*; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -818,7 +819,7 @@ public void setRecordGroup( metadata.getSourceInfo().setGroupOwner(groupIdentifier); metadataManager.save(metadata); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE); new RecordGroupOwnerChangeEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), @@ -1153,7 +1154,7 @@ private void checkCanPublishToAllGroup(ServiceContext context, ResourceBundle me if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - metadataIndexer.indexMetadata(metadata.getId() + "", true, IndexingMode.full); + metadataIndexer.indexMetadata(metadata.getId() + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); } boolean isInvalid = @@ -1217,7 +1218,7 @@ private void shareMetadataWithReservedGroup(String metadataUuid, boolean publish setOperations(sharing, dataManager, context, appContext, metadata, operationMap, privileges, ApiUtils.getUserSession(session).getUserIdAsInt(), true, null, request, metadataListToNotifyPublication, notifyByEmail); - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); java.util.Optional publicationOption = publicationConfig.getPublicationOptionConfiguration(publicationType); if (publicationOption.isPresent() && diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java index 76bb7f594f3..0a56259a15e 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java @@ -52,6 +52,7 @@ import org.fao.geonet.kernel.schema.AssociatedResourcesSchemaPlugin; import org.fao.geonet.kernel.schema.SchemaPlugin; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; @@ -306,7 +307,7 @@ public static Map> getAssociated( if (!e.fields().isEmpty()) { FIELDLIST_RELATED_SCRIPTED.keySet().forEach(f -> { JsonData dc = (JsonData) e.fields().get(f); - + if (dc != null) { if (associatedRecord.getProperties() == null) { associatedRecord.setProperties(new HashMap<>()); @@ -765,7 +766,7 @@ public static boolean retrieveMetadataValidationStatus(AbstractMetadata metadata if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - dataManager.indexMetadata(metadata.getId() + "", true); + dataManager.indexMetadata(metadata.getId() + "", DirectIndexSubmittor.INSTANCE); } boolean isInvalid = diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java index fc7e699fc24..e3e185e0418 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java @@ -45,6 +45,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.GeonetworkDataDirectory; import org.fao.geonet.kernel.schema.MetadataSchema; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SchematronRepository; import org.fao.geonet.utils.IO; @@ -189,7 +190,7 @@ Reports validateRecord( .setStatus(isvalid ? MetadataValidationStatus.VALID : MetadataValidationStatus.INVALID) .setRequired(true).setNumTests(0).setNumFailures(0); this.metadataValidationRepository.save(metadataValidation); - dataManager.indexMetadata(("" + metadata.getId()), true); + dataManager.indexMetadata(("" + metadata.getId()), DirectIndexSubmittor.INSTANCE); new RecordValidationTriggeredEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), metadataValidation.getStatus().getCode()).publish(appContext); diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java index dd72b20e8fe..1a14d3e07e0 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java @@ -58,6 +58,7 @@ import org.fao.geonet.kernel.metadata.StatusChangeType; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -517,7 +518,7 @@ public Map setStatus(@Parameter(description = API_PAR if (statusUpdate.get(metadata.getId()) == StatusChangeType.UPDATED) { //--- reindex metadata - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); // Reindex the metadata table record to update the field _statusWorkflow that contains the composite // status of the published and draft versions @@ -526,7 +527,7 @@ public Map setStatus(@Parameter(description = API_PAR if (metadataApproved != null) { metadataIdApproved = metadataApproved.getId(); - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } } } @@ -945,7 +946,7 @@ public void restoreAtStatusSave( recoveredMetadataId = reloadRecord(element, metadataManager, httpSession, request); } - metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), DirectIndexSubmittor.INSTANCE, IndexingMode.full); UserSession session = ApiUtils.getUserSession(request.getSession()); if (session != null) { diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java index cf06d7c9e34..33a4ddd3e71 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java @@ -37,6 +37,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.schema.SchemaPlugin; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.lib.Lib; import org.fao.geonet.schema.iso19139.ISO19139Namespaces; import org.fao.geonet.utils.Log; @@ -695,7 +696,7 @@ public synchronized boolean addAttribute(String id, String ref, String name, Str String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, DirectIndexSubmittor.INSTANCE); return true; } @@ -740,7 +741,7 @@ public synchronized boolean deleteAttribute(String id, String ref, String name, String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, DirectIndexSubmittor.INSTANCE); return true; } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java index 16aaee8d1d5..2f50c957bc1 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java @@ -54,6 +54,7 @@ import org.fao.geonet.kernel.metadata.StatusActions; import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -485,7 +486,7 @@ public void saveEdits( if (reindex) { Log.trace(Geonet.DATA_MANAGER, " > Reindexing record"); - metadataIndexer.indexMetadata(id, true, IndexingMode.full); + metadataIndexer.indexMetadata(id, DirectIndexSubmittor.INSTANCE, IndexingMode.full); } // Reindex the metadata table record to update the field _statusWorkflow that contains the composite @@ -494,7 +495,7 @@ public void saveEdits( Metadata metadataApproved = metadataRepository.findOneByUuid(metadata.getUuid()); if (metadataApproved != null) { - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), true, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } } diff --git a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java index a011b58b68d..873fb2274c2 100644 --- a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java +++ b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java @@ -13,6 +13,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.util.Sha1Encoder; import org.fao.geonet.utils.Log; @@ -44,7 +45,7 @@ public static Map saveEntries(ServiceContext context, entries.rowKeySet().iterator(); AbstractMetadata record = collectResults.getRecord(); boolean validate = false, ufo = false, - notify = false, publicForGroup = true, refreshReaders = false; + notify = false, publicForGroup = true; Map errors = new HashMap<>(); while (entriesIterator.hasNext()) { @@ -73,7 +74,7 @@ public static Map saveEntries(ServiceContext context, (Element) entry.clone(), IndexingMode.none, ufo, UpdateDatestamp.NO, - publicForGroup, refreshReaders); + publicForGroup, DirectIndexSubmittor.INSTANCE); collectResults.getEntryIdentifiers().put( uuid, subtemplate.getId()); diff --git a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java index 595ec851698..04cbeee336d 100644 --- a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java +++ b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java @@ -37,6 +37,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.repository.UserGroupRepository; @@ -325,7 +327,7 @@ ResponseEntity addToUserSelection( UserSavedSelection e = new UserSavedSelection(selection.get(), user.get(), u); try { umsRepository.save(e); - metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), true, IndexingMode.full); + metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } catch (Exception e1) { Log.error(API.LOG_MODULE_NAME, "UserSelectionsApi - addToUserSelection: " + e1.getMessage(), e1); } @@ -384,13 +386,15 @@ ResponseEntity deleteFromUserSelection( if (uuid == null || uuid.length == 0) { umsRepository.deleteAllBySelectionAndUser(selectionIdentifier, userIdentifier); } else { - for (String u : uuid) { - UserSavedSelectionId e = new UserSavedSelectionId() - .setSelectionId(selectionIdentifier) - .setUserId(userIdentifier) - .setMetadataUuid(u); - umsRepository.deleteById(e); - metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), true, IndexingMode.full); + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { + for (String u : uuid) { + UserSavedSelectionId e = new UserSavedSelectionId() + .setSelectionId(selectionIdentifier) + .setUserId(userIdentifier) + .setMetadataUuid(u); + umsRepository.deleteById(e); + metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), batchingIndexSubmittor, IndexingMode.full); + } } } diff --git a/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java b/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java index 508f5cca808..bb95bebfe8e 100644 --- a/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/links/LinksApiTest.java @@ -98,7 +98,7 @@ public void getLinks() throws Exception { private void createTestData() throws Exception { loginAsAdmin(context); - this.md = injectMetadataInDb(getSampleMetadataXml(), context, true); + this.md = injectMetadataInDb(getSampleMetadataXml(), context); Element xmlData = this.md.getXmlData(false); urlAnalyzer.processMetadata(xmlData, this.md); diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java index dd9f3880916..82006ecc5c3 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataApiTest.java @@ -80,7 +80,7 @@ public void setUp() throws Exception { private void createTestData() throws Exception { loginAsAdmin(context); - AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context, true); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context); id = metadata.getId(); uuid = metadata.getUuid(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java index 4f8afb52795..d22e9da3039 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java @@ -24,6 +24,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -208,9 +209,9 @@ private AbstractMetadata subTemplateOnLineResourceDbInsert(MetadataType type) th false, NO, false, - false); + DirectIndexSubmittor.INSTANCE); - dataManager.indexMetadata("" + dbInsertedMetadata.getId(), true); + dataManager.indexMetadata("" + dbInsertedMetadata.getId(), DirectIndexSubmittor.INSTANCE); assertEquals(1, countTemplateIndexed(dbInsertedMetadata.getUuid(), false, type == MetadataType.SUB_TEMPLATE ? "s" : "n")); return dbInsertedMetadata; } diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java index a56815862c2..87ec40f129a 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataWorkflowApiTest.java @@ -56,10 +56,10 @@ public void setUp() throws Exception { settingManager.setValue(Settings.METADATA_WORKFLOW_ENABLE, true); settingManager.setValue(Settings.METADATA_WORKFLOW_DRAFT_WHEN_IN_GROUP, ".*"); - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), context); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), context); injectStatusForMetadata(metadata, DRAFT); uuid = metadata.getUuid(); - AbstractMetadata metadata2 = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), context); + AbstractMetadata metadata2 = injectMetadataInDb(getSampleMetadataXml(), context); injectStatusForMetadata(metadata2, APPROVED); uuid2 = metadata2.getUuid(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java index 20ab40561e6..462efae380e 100644 --- a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java @@ -32,6 +32,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.fao.geonet.utils.Xml; @@ -339,7 +340,7 @@ private String createMdFromXmlRessources(Element sampleMetadataXml) throws Excep .setHarvested(false); dataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.full, false, UpdateDatestamp.NO, - false, false).getId(); + false, DirectIndexSubmittor.INSTANCE).getId(); return uuid; } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java index 285b4e29f05..0d62fd75be9 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java @@ -34,6 +34,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.languages.IsoLanguagesMapper; import org.fao.geonet.repository.MetadataRepository; @@ -97,7 +98,7 @@ public void setUp() throws Exception { metadata.setHarvestInfo(harvestInfo); metadata.setData(xml); this.id = dataManager.insertMetadata(serviceContext, metadata, metadata.getXmlData(false), IndexingMode.full, false, - UpdateDatestamp.NO, false, false).getId(); + UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE).getId(); } public boolean isHarvested() { diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java index be863787a89..0209e4af23e 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/AlternateLogoForPdfExportTest.java @@ -53,7 +53,7 @@ public void initSiteId() { public void createTestData() throws Exception { context = createServiceContext(); loginAsAdmin(context); - metadata = injectMetadataInDbDoNotRefreshHeader(getSampleISO19139MetadataXml(), context); + metadata = injectMetadataInDb(getSampleISO19139MetadataXml(), context); } @Before diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java index 60bf0c6a49f..813073d5f62 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiIntegrationTest.java @@ -75,7 +75,7 @@ public class FormatterApiIntegrationTest extends AbstractServiceIntegrationTest public void setUp() throws Exception { serviceContext = createServiceContext(); loginAsAdmin(serviceContext); - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(getSampleMetadataXml(), serviceContext); + AbstractMetadata metadata = injectMetadataInDb(getSampleMetadataXml(), serviceContext); id = metadata.getId(); schema = metadata.getDataInfo().getSchemaId(); } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java index 4d0f3c79c31..a9c1b8293c6 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/FormatterApiTest.java @@ -298,7 +298,7 @@ private void createTestData() throws Exception { } private void loadFile(String key, Element sampleMetadataXml) throws Exception { - AbstractMetadata metadata = injectMetadataInDbDoNotRefreshHeader(sampleMetadataXml, context); + AbstractMetadata metadata = injectMetadataInDb(sampleMetadataXml, context); testDataUuidBySchema.put(key, metadata.getUuid()); } } diff --git a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java index 9cdd6ed90d9..b2684f22a92 100644 --- a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java @@ -32,6 +32,7 @@ import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; @@ -51,8 +52,7 @@ import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -229,7 +229,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isCreated()); - verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), any(Boolean.class), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), eq(DirectIndexSubmittor.INSTANCE), eq(IndexingMode.full)); this.mockMvc.perform(get("/srv/api/userselections/" + createdSelection.getId() + "/1") .session(this.mockHttpSession) @@ -244,7 +244,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isNoContent()); - verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(Boolean.class), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), eq(DirectIndexSubmittor.INSTANCE), eq(IndexingMode.full)); // Delete this.mockMvc.perform(delete("/srv/api/userselections/" + createdSelection.getId()) diff --git a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java index d8ec3f798b4..293dd9c6f72 100644 --- a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java +++ b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java @@ -5,6 +5,8 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.ISearchManager; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.util.ThreadUtils; import org.junit.Test; import org.junit.runner.RunWith; @@ -192,7 +194,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { usedTread.add(Thread.currentThread()); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.anyBoolean()); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmittor.class)); return mockDataMan; } @@ -206,7 +208,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { latch.await(); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.anyBoolean()); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmittor.class)); return mockDataMan; } @@ -221,10 +223,10 @@ private Set createMetadataToIndex() { private ArgumentCaptor captureIndexationLaunched(DataManager mockDataMan) throws Exception { ArgumentCaptor metadataIdCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor forceRefreshCaptor = ArgumentCaptor.forClass(Boolean.class); + ArgumentCaptor indexSubmittorCaptor = ArgumentCaptor.forClass(IIndexSubmittor.class); ArgumentCaptor isearchManagerCaptor = ArgumentCaptor.forClass(ISearchManager.class); - Mockito.verify(mockDataMan, Mockito.times(4)).indexMetadata(metadataIdCaptor.capture(), forceRefreshCaptor.capture()); + Mockito.verify(mockDataMan, Mockito.times(4)).indexMetadata(metadataIdCaptor.capture(), indexSubmittorCaptor.capture()); return metadataIdCaptor; } } From 418de0a6b69edd3dbc5d719462a9543ea828d990 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Wed, 30 Oct 2024 13:26:12 +0100 Subject: [PATCH 03/18] Dynamically compute the batch size based on the number of expected index entries --- .../fao/geonet/kernel/IndexMetadataTask.java | 2 +- .../datamanager/base/BaseMetadataIndexer.java | 2 +- .../geonet/kernel/search/EsSearchManager.java | 52 ++++++++++--------- .../index/BatchOpsMetadataReindexer.java | 2 +- .../kernel/search/index/IndexingTask.java | 11 ++-- .../submission/BatchingIndexSubmittor.java | 27 +++++++++- .../api/selections/UserSelectionsApi.java | 2 +- 7 files changed, 62 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index 663b5e50105..1d5b8888cca 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -78,7 +78,7 @@ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metad } public void run() { - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(_metadataIds.size())) { _context.setAsThreadLocal(); while (_transactionStatus != null && !_transactionStatus.isCompleted()) { try { diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 37162e0b5da..330b21627f2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -302,7 +302,7 @@ public boolean isIndexing() { @Override public void indexMetadata(final List metadataIds) throws Exception { - try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor()) { + try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(metadataIds.size())) { for (String metadataId : metadataIds) { indexMetadata(metadataId, indexSubmittor, IndexingMode.full); } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 50f23bce2ee..c929783367d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -717,38 +717,40 @@ public boolean rebuildIndex(ServiceContext context, clearIndex(); } - try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor()) { - if (StringUtils.isNotBlank(bucket)) { - ArrayList listOfIdsToIndex = new ArrayList<>(); - UserSession session = context.getUserSession(); - SelectionManager sm = SelectionManager.getManager(session); - - synchronized (sm.getSelection(bucket)) { - for (String uuid : sm.getSelection(bucket)) { - for (AbstractMetadata metadata : metadataRepository.findAllByUuid(uuid)) { - String indexKey = uuid; - if (metadata instanceof MetadataDraft) { - indexKey += "-draft"; - } - - listOfIdsToIndex.add(indexKey); + if (StringUtils.isNotBlank(bucket)) { + ArrayList listOfIdsToIndex = new ArrayList<>(); + UserSession session = context.getUserSession(); + SelectionManager sm = SelectionManager.getManager(session); + + synchronized (sm.getSelection(bucket)) { + for (String uuid : sm.getSelection(bucket)) { + for (AbstractMetadata metadata : metadataRepository.findAllByUuid(uuid)) { + String indexKey = uuid; + if (metadata instanceof MetadataDraft) { + indexKey += "-draft"; } - if (!metadataRepository.existsMetadataUuid(uuid)) { - LOGGER.warn("Selection contains uuid '{}' not found in database", uuid); - } + listOfIdsToIndex.add(indexKey); + } + + if (!metadataRepository.existsMetadataUuid(uuid)) { + LOGGER.warn("Selection contains uuid '{}' not found in database", uuid); } } + } + try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(listOfIdsToIndex.size())) { for (String id : listOfIdsToIndex) { metadataIndexer.indexMetadata(id, indexSubmittor, IndexingMode.full); } - } else { - final Specification metadataSpec = - Specification.where((Specification) MetadataSpecs.isType(MetadataType.METADATA)) - .or((Specification) MetadataSpecs.isType(MetadataType.TEMPLATE)); - final List metadataIds = metadataRepository.findAllIdsBy( - Specification.where(metadataSpec) - ); + } + } else { + final Specification metadataSpec = + Specification.where((Specification) MetadataSpecs.isType(MetadataType.METADATA)) + .or((Specification) MetadataSpecs.isType(MetadataType.TEMPLATE)); + final List metadataIds = metadataRepository.findAllIdsBy( + Specification.where(metadataSpec) + ); + try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(metadataIds.size())) { for (Integer id : metadataIds) { metadataIndexer.indexMetadata(id + "", indexSubmittor, IndexingMode.full); } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index c782cefb392..12a41ab2564 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -209,7 +209,7 @@ public void run() { "Indexing range [%d-%d]/%d by threads %s.", beginIndex, beginIndex + count, ids.length, Thread.currentThread().getId())); long start = System.currentTimeMillis(); - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(count)) { for (int i = beginIndex; i < beginIndex + count; i++) { try { dm.indexMetadata(ids[i] + "", batchingIndexSubmittor); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index 1166a8e8108..a5a57aec570 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -68,11 +68,12 @@ private void indexRecords() { return; } - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { - if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) { - Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " - + metadataIdentifiers.toString() + "."); - } + + if (Log.isDebugEnabled(Geonet.INDEX_ENGINE)) { + Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " + + metadataIdentifiers.toString() + "."); + } + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(metadataIdentifiers.size())) { for (Integer metadataIdentifier : metadataIdentifiers) { try { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java index 02c5d71eac8..8085b1b803f 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java @@ -24,14 +24,37 @@ public class BatchingIndexSubmittor implements AutoCloseable, IIndexSubmittor { * Maximum number of inflight bulk requests before waiting for the Elasticsearch */ private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; - private static final int COMMIT_INTERVAL = 200; @SuppressWarnings("unchecked") private final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; private final Map listOfDocumentsToIndex = new HashMap<>(); + private final int commitInterval; private int index; private boolean closed = false; private EsSearchManager searchManager; + public BatchingIndexSubmittor() { + this.commitInterval = 200; + } + + /** + * @param estimatedTotalSize The estimated size of documents to index. Does not need to match the actual amount of submitted documents + */ + public BatchingIndexSubmittor(int estimatedTotalSize) { + if (estimatedTotalSize < 0) { + throw new IllegalArgumentException("estimatedTotalSize must not be negative"); + } + + // Compute an ideal commit interval based on estimated size of elements to index + // Try to strike a balance between + // a) Not making enough bulk requests, thus having to wait a long time at the end for a large chunk => try to make at least 8 requests + int elementsPerBatchRequest = estimatedTotalSize / 8; + // b) Making too many requests, adding unnecessary overhead => set the minimum batch size to 20 + elementsPerBatchRequest = Math.max(20, elementsPerBatchRequest); + // c) Growing the listOfDocumentsToIndex too large => set the maximum batch size to 200 + elementsPerBatchRequest = Math.min(200, elementsPerBatchRequest); + this.commitInterval = elementsPerBatchRequest; + } + @Override public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { if (closed) { @@ -40,7 +63,7 @@ public void submitToIndex(String id, String jsonDocument, EsSearchManager search this.searchManager = searchManager; listOfDocumentsToIndex.put(id, jsonDocument); - if (listOfDocumentsToIndex.size() >= COMMIT_INTERVAL) { + if (listOfDocumentsToIndex.size() >= commitInterval) { Map toIndex = new HashMap<>(listOfDocumentsToIndex); listOfDocumentsToIndex.clear(); sendDocumentsToIndex(toIndex); diff --git a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java index 04cbeee336d..cad896ad665 100644 --- a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java +++ b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java @@ -386,7 +386,7 @@ ResponseEntity deleteFromUserSelection( if (uuid == null || uuid.length == 0) { umsRepository.deleteAllBySelectionAndUser(selectionIdentifier, userIdentifier); } else { - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor()) { + try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(uuid.length)) { for (String u : uuid) { UserSavedSelectionId e = new UserSavedSelectionId() .setSelectionId(selectionIdentifier) From 9519a92b0f5f0918785e92254939872fd98ffd2a Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Thu, 31 Oct 2024 15:50:32 +0100 Subject: [PATCH 04/18] Use a cleaner to make sure all documents get send to the index, even in case on of not properly closed submittors --- .../submission/BatchingIndexSubmittor.java | 115 ++++++++++-------- 1 file changed, 67 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java index 8085b1b803f..e82a0d762bf 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.lang.ref.Cleaner; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -20,20 +21,71 @@ */ public class BatchingIndexSubmittor implements AutoCloseable, IIndexSubmittor { private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.INDEX_ENGINE); + private static final Cleaner CLEANER = Cleaner.create(); /** * Maximum number of inflight bulk requests before waiting for the Elasticsearch */ private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; - @SuppressWarnings("unchecked") - private final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; - private final Map listOfDocumentsToIndex = new HashMap<>(); + + private static class State implements Runnable { + private final Map listOfDocumentsToIndex = new HashMap<>(); + @SuppressWarnings("unchecked") + private final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; + private int index; + private EsSearchManager searchManager; + private boolean closed = false; + + @Override + public void run() { + if (!closed) { + LOGGER.error("BatchingIndexSubmittor was not closed before it was cleaned! Sending any remaining data"); + } + // Send any remaining pending documents + if (!listOfDocumentsToIndex.isEmpty()) { + sendDocumentsToIndex(listOfDocumentsToIndex); + } + } + + + private void sendDocumentsToIndex(Map toIndex) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), listOfDocumentsToIndex); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + try { + searchManager.handleIndexResponse(bulkItemResponses, listOfDocumentsToIndex); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).exceptionally(e -> { + LOGGER.error( + "An error occurred while indexing {} documents in current indexing list.", + toIndex.size(), e); + return null; + }); + + // Request send, sort into queue + // First, see if the previous future already finished + CompletableFuture previousFuture = inflightFutures[index]; + if (previousFuture != null) { + if (!previousFuture.isDone()) { + // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer + LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); + previousFuture.join(); + } + } + inflightFutures[index] = currentIndexFuture; + index = (index + 1) % inflightFutures.length; + } + } + + private final State state = new State(); + private final Cleaner.Cleanable cleanable; private final int commitInterval; - private int index; - private boolean closed = false; - private EsSearchManager searchManager; public BatchingIndexSubmittor() { this.commitInterval = 200; + this.cleanable = CLEANER.register(this, state); } /** @@ -53,71 +105,38 @@ public BatchingIndexSubmittor(int estimatedTotalSize) { // c) Growing the listOfDocumentsToIndex too large => set the maximum batch size to 200 elementsPerBatchRequest = Math.min(200, elementsPerBatchRequest); this.commitInterval = elementsPerBatchRequest; + this.cleanable = CLEANER.register(this, state); } @Override public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { - if (closed) { + if (state.closed) { throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); } - this.searchManager = searchManager; + state.searchManager = searchManager; + Map listOfDocumentsToIndex = state.listOfDocumentsToIndex; listOfDocumentsToIndex.put(id, jsonDocument); if (listOfDocumentsToIndex.size() >= commitInterval) { Map toIndex = new HashMap<>(listOfDocumentsToIndex); listOfDocumentsToIndex.clear(); - sendDocumentsToIndex(toIndex); + state.sendDocumentsToIndex(toIndex); } } @Override public void close() { - if (closed) { + if (this.state.closed) { throw new IllegalStateException("Attempting to close a already closed " + this.getClass().getSimpleName()); } - this.closed = true; - - // Send any remaining pending documents - if (!this.listOfDocumentsToIndex.isEmpty()) { - sendDocumentsToIndex(this.listOfDocumentsToIndex); - } + this.state.closed = true; + this.cleanable.clean(); // Wait for all remaining documents to be received - for (CompletableFuture inflightFuture : inflightFutures) { + for (CompletableFuture inflightFuture : state.inflightFutures) { if (inflightFuture != null) { inflightFuture.join(); } } } - - private void sendDocumentsToIndex(Map toIndex) { - EsRestClient restClient = searchManager.getClient(); - BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), listOfDocumentsToIndex); - CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) - .thenAcceptAsync(bulkItemResponses -> { - try { - searchManager.handleIndexResponse(bulkItemResponses, listOfDocumentsToIndex); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).exceptionally(e -> { - LOGGER.error( - "An error occurred while indexing {} documents in current indexing list.", - toIndex.size(), e); - return null; - }); - - // Request send, sort into queue - // First, see if the previous future already finished - CompletableFuture previousFuture = inflightFutures[index]; - if (previousFuture != null) { - if (!previousFuture.isDone()) { - // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer - LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); - previousFuture.join(); - } - } - inflightFutures[index] = currentIndexFuture; - index = (index + 1) % inflightFutures.length; - } } From a9a919e62fc2c09dbce61cfc9a82354045eb7312 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Thu, 31 Oct 2024 15:57:52 +0100 Subject: [PATCH 05/18] Don't use list version of indexMetadata for single entries --- .../kernel/search/submission/BatchingIndexSubmittor.java | 1 - .../kernel/search/submission/DirectIndexSubmittor.java | 1 + .../geonet/inspireatom/harvester/InspireAtomHarvester.java | 7 ++++--- .../fao/geonet/listener/metadata/draft/DraftCreated.java | 4 +++- .../fao/geonet/listener/metadata/draft/DraftRemoved.java | 4 +++- .../fao/geonet/listener/metadata/draft/RetireRecord.java | 4 +++- .../geonet/listener/metadata/draft/UpdateOperations.java | 4 +++- .../inspire/validator/InspireValidationRunnable.java | 3 ++- .../inspire/validator/MInspireEtfValidateProcess.java | 3 ++- 9 files changed, 21 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java index e82a0d762bf..559b405d47d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java @@ -46,7 +46,6 @@ public void run() { } } - private void sendDocumentsToIndex(Map toIndex) { EsRestClient restClient = searchManager.getClient(); BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), listOfDocumentsToIndex); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java index e6500bb4e42..6d628b72cf0 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java @@ -23,6 +23,7 @@ public void submitToIndex(String id, String jsonDocument, EsSearchManager search Map documents = Collections.singletonMap(id, jsonDocument); BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), documents); + // TODO log any exception final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); searchManager.handleIndexResponse(bulkItemResponses, documents); diff --git a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java index 7954f45fd67..8231155be4f 100644 --- a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java +++ b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java @@ -37,6 +37,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.InspireAtomFeedRepository; @@ -169,7 +170,7 @@ public final void harvestServiceMetadata(final ServiceContext context, final Str // Removes all atom information from existing metadata. Harvester will reload with updated information localLogger.info("ATOM feed harvest: remove existing metadata feed"); repository.deleteAll(InspireAtomFeedSpecs.hasMetadataId(Integer.parseInt(metadataId))); - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); // Process service metadata feeds // datasetsInformation stores the dataset information for identifier and namespace for the services feed. @@ -244,7 +245,7 @@ private List processServiceMetadataFeeds(final DataManager data repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); // Extract datasets information (identifier, namespace) from the service feed: @@ -337,7 +338,7 @@ private void processDatasetsMetadataFeeds(final DataManager dataMan, repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(Arrays.asList(metadataId)); + dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); result.addContent(new Element("feed").setAttribute("uuid", metadataUuid) .setAttribute("feed", atomUrl).setAttribute("status", "ok")); } catch (Exception ex) { diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java index 83d5458e7a6..b71164128c1 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java @@ -31,6 +31,8 @@ import org.fao.geonet.events.md.MetadataDraftAdd; import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -67,7 +69,7 @@ public void doAfterCommit(MetadataDraftAdd event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(md.getId()))); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java index bc957e84554..3a2d5839650 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java @@ -35,6 +35,8 @@ import org.fao.geonet.events.md.MetadataDraftRemove; import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -67,7 +69,7 @@ public void doAfterCommit(MetadataDraftRemove event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(md.getId()))); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java index 98fe1a244ad..188df98b695 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java @@ -33,6 +33,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -88,7 +90,7 @@ public void doBeforeCommit(MetadataStatusChanged event) { metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.view.getId()); metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.notify.getId()); - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(event.getMd().getId()))); + metadataIndexer.indexMetadata(String.valueOf(event.getMd().getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error upgrading status", e); diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java index fb6049666c1..6346a483a28 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java @@ -36,6 +36,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.utils.Log; @@ -121,7 +123,7 @@ public void doAfterCommit(MetadataShare event) { metadataOperations.forceSetOperation(context, draft.getId(), op.getId().getGroupId(), op.getId().getOperationId()); } - metadataIndexer.indexMetadata(Arrays.asList(String.valueOf(draft.getId()))); + metadataIndexer.indexMetadata(String.valueOf(draft.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error cascading operation to draft", e); } diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java index ab8d64557b6..297f181abc7 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java @@ -31,6 +31,7 @@ import org.fao.geonet.domain.MetadataValidationId; import org.fao.geonet.domain.MetadataValidationStatus; import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.MetadataValidationRepository; import org.springframework.transaction.TransactionStatus; @@ -91,7 +92,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { DataManager dataManager = ApplicationContextHolder.get().getBean(DataManager.class); - dataManager.indexMetadata(new ArrayList<>(Arrays.asList(mdId + ""))); + dataManager.indexMetadata(mdId + "", DirectIndexSubmittor.INSTANCE); return null; } diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java index 07c1faf1c71..a74d53f2a1c 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java @@ -16,6 +16,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -263,7 +264,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { } if (reindexMetadata) { - dataManager.indexMetadata(new ArrayList<>(Arrays.asList(metadataRecord.getId() + ""))); + dataManager.indexMetadata(metadataRecord.getId() + "", DirectIndexSubmittor.INSTANCE); } } catch (Exception ex) { From 0d49d86865325b24b450aed635c47f141fb33b38 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Thu, 31 Oct 2024 16:37:57 +0100 Subject: [PATCH 06/18] Remove no longer required method forceIndexChanges --- .../org/fao/geonet/kernel/DataManager.java | 5 --- .../fao/geonet/kernel/IndexMetadataTask.java | 10 +----- .../kernel/datamanager/IMetadataIndexer.java | 5 --- .../datamanager/base/BaseMetadataIndexer.java | 8 +---- .../geonet/kernel/search/EsSearchManager.java | 5 --- .../geonet/kernel/search/ISearchManager.java | 6 ---- .../kernel/harvest/harvester/csw/Aligner.java | 1 - .../harvest/harvester/geoPREST/Aligner.java | 2 -- .../harvest/harvester/oaipmh/Harvester.java | 1 - .../harvest/harvester/simpleurl/Aligner.java | 1 - .../api/records/MetadataInsertDeleteApi.java | 2 -- .../java/org/fao/geonet/api/site/SiteApi.java | 34 ------------------- 12 files changed, 2 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index 00e6a930e7d..b60d660fb6b 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -579,11 +579,6 @@ public void flush() { metadataManager.flush(); } - @Deprecated - public void forceIndexChanges() throws IOException { - metadataIndexer.forceIndexChanges(); - } - @Deprecated public int batchDeleteMetadataAndUpdateIndex(Specification specification) throws Exception { return metadataIndexer.batchDeleteMetadataAndUpdateIndex(specification); diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index 1d5b8888cca..f74f10be15e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -50,8 +50,6 @@ public final class IndexMetadataTask implements Runnable { private final List _metadataIds; private final TransactionStatus _transactionStatus; private final Set _batchIndex; - private final EsSearchManager searchManager; - private final AtomicInteger indexed; private User _user; /** @@ -62,13 +60,11 @@ public final class IndexMetadataTask implements Runnable { * @param transactionStatus if non-null, wait for the transaction to complete before indexing */ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metadataIds, Set batchIndex, - @Nullable TransactionStatus transactionStatus, @Nonnull AtomicInteger indexed) { - this.indexed = indexed; + @Nullable TransactionStatus transactionStatus) { this._transactionStatus = transactionStatus; this._context = context; this._metadataIds = metadataIds; this._batchIndex = batchIndex; - this.searchManager = context.getBean(EsSearchManager.class); batchIndex.add(this); @@ -102,10 +98,6 @@ public void run() { DataManager dataManager = _context.getBean(DataManager.class); // servlet up so safe to index all metadata that needs indexing for (Object metadataId : _metadataIds) { - this.indexed.incrementAndGet(); - if (this.indexed.compareAndSet(500, 0)) { - searchManager.forceIndexChanges(); - } try { dataManager.indexMetadata(metadataId.toString(), batchingIndexSubmittor); diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java index 09d9c76e5a6..cc8315f4f87 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java @@ -50,11 +50,6 @@ public interface IMetadataIndexer { */ public void init(ServiceContext context, Boolean force) throws Exception; - /** - * Force the index to wait until all changes are processed and the next reader obtained will get the latest data. - */ - void forceIndexChanges() throws IOException; - /** * Remove the records that matches the specification * diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 330b21627f2..a5965069037 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -149,11 +149,6 @@ public void setMetadataManager(IMetadataManager metadataManager) { Set indexing = new HashSet(); Set batchIndex = ConcurrentHashMap.newKeySet(); - @Override - public void forceIndexChanges() throws IOException { - searchManager.forceIndexChanges(); - } - @Override public int batchDeleteMetadataAndUpdateIndex(Specification specification) throws Exception { @@ -270,7 +265,6 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) Log.debug(Geonet.INDEX_ENGINE, "Indexing " + metadataIds.size() + " records."); Log.debug(Geonet.INDEX_ENGINE, metadataIds.toString()); } - AtomicInteger numIndexedTracker = new AtomicInteger(); while (index < metadataIds.size()) { int start = index; int count = Math.min(perThread, metadataIds.size() - start); @@ -287,7 +281,7 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) } // create threads to process this chunk of ids - Runnable worker = new IndexMetadataTask(context, subList, batchIndex, transactionStatus, numIndexedTracker); + Runnable worker = new IndexMetadataTask(context, subList, batchIndex, transactionStatus); executor.execute(worker); index += count; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index c929783367d..a1433fbc244 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -702,11 +702,6 @@ private String parseBoolean(String value) { return String.valueOf(booleanValues.contains(value)); } - @Override - public void forceIndexChanges() { - // TODO remove - no longer needed due to new indexing behaviour - } - @Override public boolean rebuildIndex(ServiceContext context, boolean reset, String bucket) throws Exception { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index 67e40b64e8f..9f4a39cca75 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -54,12 +54,6 @@ void index(Path schemaDir, Element metadata, String id, Multimap MetadataType metadataType, IIndexSubmittor indexSubmittor, IndexingMode indexingMode) throws Exception; - /** - * Force the index to wait until all changes are processed and the next reader obtained will get - * the latest data. - */ - void forceIndexChanges() throws IOException; - /** * Rebuilds the Lucene index. If xlink or from selection parameters are defined, reindex a diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 50388cc5044..6408ba24fdd 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -264,7 +264,6 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception result.locallyRemoved++; } } - dataMan.forceIndexChanges(); return result; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index 153f6bdec24..a82e1ccfd04 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -145,8 +145,6 @@ public HarvestResult align(Set records, List errors) t } } - dataMan.forceIndexChanges(); - log.info("End of alignment for : " + params.getName()); return result; diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java index 9871f5aeecc..d132c111137 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java @@ -353,7 +353,6 @@ private void align(XmlRequest t, Set records) throws Exception { } } - dataMan.forceIndexChanges(); log.info("End of alignment for : " + params.getName()); } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java index a7ca4eb1874..72b5ed23299 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java @@ -186,7 +186,6 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception result.locallyRemoved ++; } } - dataMan.forceIndexChanges(); return result; } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index 2d00792fdce..cf78db08ec3 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -244,8 +244,6 @@ public void deleteRecord( RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, metadata.getId() + ""); metadataManager.deleteMetadata(context, metadata.getId() + ""); recordDeletedEvent.publish(ApplicationContextHolder.get()); - - dataManager.forceIndexChanges(); } @io.swagger.v3.oas.annotations.Operation(summary = "Delete one or more records", description ="User MUST be able to edit the record to delete it. " diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java index 01882b580ed..ed3154ea1d7 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java @@ -614,22 +614,6 @@ public HttpEntity indexSite( return new HttpEntity<>(HttpStatus.CREATED); } - @io.swagger.v3.oas.annotations.Operation( - summary = "Index commit", - description = "") - @RequestMapping( - path = "/index/commit", - produces = MediaType.APPLICATION_JSON_VALUE, - method = RequestMethod.GET) - @ResponseStatus(HttpStatus.OK) - @PreAuthorize("hasAuthority('Administrator')") - public void indexCommit( - HttpServletRequest request - ) throws Exception { - EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); - searchMan.forceIndexChanges(); - } - @io.swagger.v3.oas.annotations.Operation( summary = "Index status", @@ -676,24 +660,6 @@ public Map indexAndDbSynchronizationStatus( } - @io.swagger.v3.oas.annotations.Operation( - summary = "Force to commit pending documents in index.", - description = "May be used when indexing task is hanging.") - @PutMapping( - path = "/index/commit") - @ApiResponses(value = { - @ApiResponse(responseCode = "201", description = "Changes committed.") - }) - @ResponseStatus(HttpStatus.CREATED) - @PreAuthorize("hasAuthority('Administrator')") - public void commitIndexChanges( - ) throws Exception { - ApplicationContextHolder.get() - .getBean(EsSearchManager.class) - .forceIndexChanges(); - } - - @io.swagger.v3.oas.annotations.Operation( summary = "Get build details", description = "To know when and how this version of the application was built.") From ff5c5dc6b9e280fc100ba7495e5f222e5d0f3ceb Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Tue, 5 Nov 2024 09:59:41 +0100 Subject: [PATCH 07/18] Remove commit index changes from frontend --- .../catalog/js/admin/AdminToolsController.js | 16 ---------------- .../main/resources/catalog/locales/az-v4.json | 3 --- .../main/resources/catalog/locales/ca-v4.json | 3 --- .../main/resources/catalog/locales/cs-v4.json | 3 --- .../main/resources/catalog/locales/cy-v4.json | 3 --- .../main/resources/catalog/locales/da-v4.json | 3 --- .../main/resources/catalog/locales/de-v4.json | 3 --- .../main/resources/catalog/locales/en-v4.json | 3 --- .../main/resources/catalog/locales/es-v4.json | 3 --- .../main/resources/catalog/locales/fi-v4.json | 3 --- .../main/resources/catalog/locales/fr-v4.json | 3 --- .../main/resources/catalog/locales/hy-v4.json | 3 --- .../main/resources/catalog/locales/is-v4.json | 3 --- .../main/resources/catalog/locales/it-v4.json | 3 --- .../main/resources/catalog/locales/ka-v4.json | 3 --- .../main/resources/catalog/locales/ko-v4.json | 3 --- .../main/resources/catalog/locales/nl-v4.json | 3 --- .../main/resources/catalog/locales/pt-v4.json | 3 --- .../main/resources/catalog/locales/ro-v4.json | 3 --- .../main/resources/catalog/locales/ru-v4.json | 3 --- .../main/resources/catalog/locales/sk-v4.json | 3 --- .../main/resources/catalog/locales/sv-v4.json | 3 --- .../main/resources/catalog/locales/uk-v4.json | 3 --- .../main/resources/catalog/locales/zh-v4.json | 3 --- .../catalog/templates/admin/tools/index.html | 16 ---------------- 25 files changed, 101 deletions(-) diff --git a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js index 393fc0268f1..148b1ccffb9 100644 --- a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js @@ -271,22 +271,6 @@ ); }; - $scope.commitIndexChanges = function (dropFirst) { - return $http.put("../api/site/index/commit").then( - function (response) { - checkIsIndexing(); - }, - function (response) { - $rootScope.$broadcast("StatusUpdated", { - title: $translate.instant("indexCommitError"), - error: response.data, - timeout: 0, - type: "danger" - }); - } - ); - }; - $scope.clearJsCache = function () { return $http.get("../../static/wroAPI/reloadModel").then(function (response) { $http.get("../../static/wroAPI/reloadCache").then(function (response) { diff --git a/web-ui/src/main/resources/catalog/locales/az-v4.json b/web-ui/src/main/resources/catalog/locales/az-v4.json index 959f7e280ef..c0b8bab5694 100644 --- a/web-ui/src/main/resources/catalog/locales/az-v4.json +++ b/web-ui/src/main/resources/catalog/locales/az-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "İndeksi yenidən qurarkən, axtarış natamam nəticələr verə bilər və CSW GetRecords əməliyyatı deaktiv edilə bilər (əgər parametrlərdə seçimi seçmisinizsə). Kataloq trafiki az olduqda bu funksiyadan istifadə edin. Verilənlər bazasında birbaşa dəyişikliklər edərkən indeksi buradan əl ilə yenidən qurmaq tövsiyə olunur. İndeks xəritələşdirilməsini dəyişdirsəniz (müq. records.json ), onda siz "İndeksi və yenidən indeksi sil" üzərinə klikləməlisiniz.", "indexInEsDoneError": "İndeksdə xəta var. Təfərrüatlar üçün qeydlərə baxın", "indexInEsDone": "İndeksləmə əməliyyatı uğurla keçdi", - "indexCommit": "İndeks dəyişikliklərini qəbul edin", - "indexCommit-help": "Yalnız indeksləşdirmə tapşırığı asılı olduqda istifadə etmək üçün.", - "indexCommitError": "İndeks dəyişiklikləri edərkən xəta baş verdi.", "ui-moreLikeThisConfig": "Bu konfiqurasiyaya daha çox bənzəyir", "ui-moreLikeThisConfig-help": "Konfiqurasiyada oxşar qeydləri axtarmaq üçün qeyd başlığı ilə təyin olunacaq more_like_this.like olmalıdır.", "ui-autocompleteConfig": "Avtomatik tamamlama konfiqurasiyası", diff --git a/web-ui/src/main/resources/catalog/locales/ca-v4.json b/web-ui/src/main/resources/catalog/locales/ca-v4.json index e54c2ab9c57..a90f18808b8 100644 --- a/web-ui/src/main/resources/catalog/locales/ca-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ca-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/cs-v4.json b/web-ui/src/main/resources/catalog/locales/cs-v4.json index 46e448ca625..a5982ee8f3d 100644 --- a/web-ui/src/main/resources/catalog/locales/cs-v4.json +++ b/web-ui/src/main/resources/catalog/locales/cs-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Při obnovování indexu může vyhledávání vracet neúplné výsledky a operace CSR GetRecords může být deaktivována (pokud jste v nastavení vybrali tuto možnost). Tuto funkci použijte, pokud je provoz katalogu nízký. Při provádění změn v databázi se doporučuje odtud znovu vytvořit index ručně. Pokud změníte mapování indexů (srov. records.json), musíte kliknout na 'Smazat index a reindexovat'.", "indexInEsDoneError": "S indexem došlo k chybě. Podrobnosti najdete v protokolech", "indexInEsDone": "Operace indexování byla úspěšná", - "indexCommit": "Potvrdit změny indexu", - "indexCommit-help": "Použít pouze v případě, že úloha indexování visí.", - "indexCommitError": "Chyba při provádění změn indexu.", "ui-moreLikeThisConfig": "Spíše jako tato konfigurace", "ui-moreLikeThisConfig-help": "Konfigurace musí mít more_like_this.like, které budou nastaveny s názvem záznamu pro vyhledávání podobných záznamů.", "ui-autocompleteConfig": "Konfigurace automatického doplňování", diff --git a/web-ui/src/main/resources/catalog/locales/cy-v4.json b/web-ui/src/main/resources/catalog/locales/cy-v4.json index e412c79f700..4cbb0538abe 100644 --- a/web-ui/src/main/resources/catalog/locales/cy-v4.json +++ b/web-ui/src/main/resources/catalog/locales/cy-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/da-v4.json b/web-ui/src/main/resources/catalog/locales/da-v4.json index 079d11dee2e..2c4ddb2bec6 100644 --- a/web-ui/src/main/resources/catalog/locales/da-v4.json +++ b/web-ui/src/main/resources/catalog/locales/da-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Mens indekset genopbygges, kan søgningen returnere ufuldstændige resultater, og CSW GetRecords-operationen kan deaktiveres (hvis du valgte denne mulighed i indstillingerne). Benyt denne funktion, når katalogtrafikken er lav. Det anbefales at genopbygge indekset manuelt herfra, når du foretager ændringer direkte i databasen. Hvis du ændrer indekskortlægning (jf . records.json ), så skal du klikke på 'Slet indeks og genindekser'.", "indexInEsDoneError": "Der er eb fejl i indekset. Se loggen for yderligere detaljer", "indexInEsDone": "Indekserings-operationen var succesfuld ", - "indexCommit": "Foretag ændringer i indekset", - "indexCommit-help": "Anvendes kun, hvis indeksering er kørt fast.", - "indexCommitError": "Fejl ved anvendelse af ændringer i indeks", "ui-moreLikeThisConfig": "Mere som denne konfiguration", "ui-moreLikeThisConfig-help": "Konfigurationen skal have tagget 'more_like_this.like', som bliver sat sammen med instansens titel, for at søge efter lignende instanser.", "ui-autocompleteConfig": "Konfiguration af autocomplete", diff --git a/web-ui/src/main/resources/catalog/locales/de-v4.json b/web-ui/src/main/resources/catalog/locales/de-v4.json index 163db0f6074..ef0682f3417 100644 --- a/web-ui/src/main/resources/catalog/locales/de-v4.json +++ b/web-ui/src/main/resources/catalog/locales/de-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Während der Neuerstellung des Index gibt die Suche möglicherweise unvollständige Ergebnisse zurück und CSW GetRecords kann deaktiviert werden (wenn dies in den Einstellungen ausgewählt wird). Verwenden Sie die Funktion, wenn der Katalogverkehr gering ist. Es wird empfohlen, den Index von hier aus manuell neu zu erstellen, wenn Sie Änderungen direkt in der Datenbank vornehmen. Wenn Sie die Indexzuordnung ändern (vgl. records.json), müssen Sie auf 'Index löschen und neu indexieren' klicken.", "indexInEsDoneError": "Es liegt ein Fehler im Index vor. Einzelheiten finden Sie in den Logs.", "indexInEsDone": "Die Indexbildung war erfolgreich", - "indexCommit": "Commit Index-Änderungen", - "indexCommit-help": "Nur verwenden, wenn die Indexbildung hängt.", - "indexCommitError": "Fehler beim Schreiben von Indexänderungen.", "ui-moreLikeThisConfig": "Besser / ähnliche wie diese Konfiguration", "ui-moreLikeThisConfig-help": "Die Konfiguration muss ein more_like_this.like haben, das mit dem Datensatztitel festgelegt wird, um nach ähnlichen Datensätzen zu suchen.", "ui-autocompleteConfig": "Konfiguration Auto-Vervollständigen", diff --git a/web-ui/src/main/resources/catalog/locales/en-v4.json b/web-ui/src/main/resources/catalog/locales/en-v4.json index 59927c5984e..98ef548227f 100644 --- a/web-ui/src/main/resources/catalog/locales/en-v4.json +++ b/web-ui/src/main/resources/catalog/locales/en-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successful", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records. (See Elasticsearch API).", "ui-moreLikeThisFilter": "More like this query filter", diff --git a/web-ui/src/main/resources/catalog/locales/es-v4.json b/web-ui/src/main/resources/catalog/locales/es-v4.json index 7e4aea85a23..9102a138e3e 100644 --- a/web-ui/src/main/resources/catalog/locales/es-v4.json +++ b/web-ui/src/main/resources/catalog/locales/es-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Cuando se está reconstruyendo el índice la búsqueda podría devolver resultados incompletos y la operación GetRecords puede mostrarse como deshabilitada (si se ha seleccionado la opción en la configuración). Usa esta función cuando el tráfico del catálogo sea bajo. Se recomienda reconstruir el índice desde aquí cuando se hayan hecho cambios directamente en la base de datos. Si se cambia la configuración del índice (fichero records.json) hay que usar 'Borrar índice y reindexar'.", "indexInEsDoneError": "Hay un error con el índice. Consulte los logs para obtener más detalles", "indexInEsDone": "La operación de indexación fue exitosa", - "indexCommit": "Confirmar los cambios del índice", - "indexCommit-help": "Usar solo si la tarea de indexación está bloqueada.", - "indexCommitError": "Error al realizar cambios en el índice.", "ui-moreLikeThisConfig": "Configuración de \"Más como éste\"", "ui-moreLikeThisConfig-help": "La configuración debe tener un more_like_this.like que se establecerá con el título del registro para buscar registros similares.", "ui-autocompleteConfig": "Configuración del autocompletado", diff --git a/web-ui/src/main/resources/catalog/locales/fi-v4.json b/web-ui/src/main/resources/catalog/locales/fi-v4.json index 7c9a0264cf0..c1688a98e8c 100644 --- a/web-ui/src/main/resources/catalog/locales/fi-v4.json +++ b/web-ui/src/main/resources/catalog/locales/fi-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/fr-v4.json b/web-ui/src/main/resources/catalog/locales/fr-v4.json index d943adf7563..7976b0a39c7 100644 --- a/web-ui/src/main/resources/catalog/locales/fr-v4.json +++ b/web-ui/src/main/resources/catalog/locales/fr-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Lors de la reconstruction de l'index, la recherche peut retourner des résultats incomplets et l'opération GetRecords du CSW peut être désactivée (selon votre configuration). Utiliser cette fonction lorsque que le traffic au niveau du catalogue est faible. Il est recommandé de reconstruire l'index après avoir effectuer des changements directement dans la base de données. Si vous modifier le schéma de l'index (cf. records.json), alors vous devez supprimer et recréer l'index.", "indexInEsDoneError": "Il y a un problème avec l'index. Vérifier les logs pour plus d'information.", "indexInEsDone": "L'indexation s'est bien déroulée.", - "indexCommit": "Committer les changements dans l'index", - "indexCommit-help": "Utiliser cette action uniquement si l'indexation reste en attente.", - "indexCommitError": "Erreur lors du commit", "ui-moreLikeThisConfig": "Configuration pour le calcul de similarité entre fiches", "ui-moreLikeThisConfig-help": "La configuration DOIT avoir un élément more_like_this.like qui sera défini avec la valeur du titre de la fiche en cours de consultation.", "ui-autocompleteConfig": "Configuration pour l'auto-completion", diff --git a/web-ui/src/main/resources/catalog/locales/hy-v4.json b/web-ui/src/main/resources/catalog/locales/hy-v4.json index 1a4572b8032..4c18b2f05f1 100644 --- a/web-ui/src/main/resources/catalog/locales/hy-v4.json +++ b/web-ui/src/main/resources/catalog/locales/hy-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Ինդեքսը վերակառուցելիս որոնումը կարող է վերադարձնել թերի արդյունքներ, և CSW GetRecords-ի գործողությունը կարող է անջատվել (եթե ընտրել եք այդ տարբերակը կարգավորումներում): Օգտագործեք այս գործառույթը, երբ կատալոգի թրաֆիկը քիչ է: Խորհուրդ է տրվում ինդեքսը ձեռքով վերակառուցել այստեղից՝ անմիջապես տվյալների բազայում փոփոխություններ կատարելիս: Եթե փոխում եք ինդեքսների քարտեզագրումը (տես records.json ), ապա պետք է սեղմեք «Ջնջել ինդեքսը և վերաինդեքսը»:", "indexInEsDoneError": "Ցուցանիշի հետ կապված սխալ կա: Մանրամասների համար տես տեղեկամատյանները", "indexInEsDone": "Ինդեքսավորման գործողությունը հաջողությամբ ավարտվեց", - "indexCommit": "Կատարել ինդեքսի փոփոխությունները", - "indexCommit-help": "Օգտագործել միայն այն դեպքում, եթե ինդեքսավորման առաջադրանքը կախված է:", - "indexCommitError": "Սխալ՝ ինդեքսի փոփոխություններ կատարելիս:", "ui-moreLikeThisConfig": "Ավելի շատ նման է այս կազմաձևմանը", "ui-moreLikeThisConfig-help": "Կազմաձևը պետք է ունենա more_like_this.like, որը կսահմանվի ռեկորդի վերնագրի հետ՝ նմանատիպ գրառումներ որոնելու համար:", "ui-autocompleteConfig": "Ավտոմատ լրացման կարգավորում", diff --git a/web-ui/src/main/resources/catalog/locales/is-v4.json b/web-ui/src/main/resources/catalog/locales/is-v4.json index c3297cfa1ab..67ccd6fa7f0 100644 --- a/web-ui/src/main/resources/catalog/locales/is-v4.json +++ b/web-ui/src/main/resources/catalog/locales/is-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/it-v4.json b/web-ui/src/main/resources/catalog/locales/it-v4.json index d5ccebd01fe..3fa974fd831 100644 --- a/web-ui/src/main/resources/catalog/locales/it-v4.json +++ b/web-ui/src/main/resources/catalog/locales/it-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/ka-v4.json b/web-ui/src/main/resources/catalog/locales/ka-v4.json index 87fdebf6ed7..a8206e848c4 100644 --- a/web-ui/src/main/resources/catalog/locales/ka-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ka-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "ინდექსის აღდგენისას, ძიებამ შეიძლება დააბრუნოს არასრული შედეგები და CSW GetRecords ოპერაცია შეიძლება გამორთოთ (თუ პარამეტრებში აირჩიეთ ვარიანტი). გამოიყენეთ ეს ფუნქცია, როდესაც კატალოგის ტრაფიკი დაბალია. რეკომენდირებულია აქედან ხელით აღადგინოთ ინდექსი, როდესაც უშუალოდ მონაცემთა ბაზაში შეიტანეთ ცვლილებები. თუ თქვენ შეცვლით ინდექსის რუკს (შდრ. records.json ), მაშინ უნდა დააჭიროთ 'Delete index and reindex'.", "indexInEsDoneError": "ინდექსთან არის შეცდომა. იხილეთ ჟურნალები დეტალებისთვის", "indexInEsDone": "ინდექსირების ოპერაცია წარმატებით დასრულდა", - "indexCommit": "ინდექსის ცვლილებების შესრულება", - "indexCommit-help": "გამოყენება მხოლოდ იმ შემთხვევაში, თუ ინდექსირების ამოცანა ჩამოკიდებულია.", - "indexCommitError": "შეცდომა ინდექსის ცვლილებების შესრულებისას.", "ui-moreLikeThisConfig": "უფრო მეტად მოსწონს ეს კონფიგურაცია", "ui-moreLikeThisConfig-help": "კონფიგურაციას უნდა ჰქონდეს more_like_this.like, რომელიც დაყენებული იქნება ჩანაწერის სათაურით მსგავსი ჩანაწერების მოსაძებნად.", "ui-autocompleteConfig": "ავტომატური დასრულების კონფიგურაცია", diff --git a/web-ui/src/main/resources/catalog/locales/ko-v4.json b/web-ui/src/main/resources/catalog/locales/ko-v4.json index 454d968b0f7..e784bffbde4 100644 --- a/web-ui/src/main/resources/catalog/locales/ko-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ko-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/nl-v4.json b/web-ui/src/main/resources/catalog/locales/nl-v4.json index 1e56731a75a..253dd7d6cb4 100644 --- a/web-ui/src/main/resources/catalog/locales/nl-v4.json +++ b/web-ui/src/main/resources/catalog/locales/nl-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Tijdens het opnieuw opbouwen van de index kan het zoeken onvolledige resultaten opleveren en kan de bewerking CSW GetRecords worden uitgeschakeld (als u de optie in de instellingen hebt geselecteerd). Gebruik deze functie als er weinig catalogusverkeer is. Het wordt aanbevolen om de index handmatig opnieuw op te bouwen vanaf hier wanneer u wijzigingen rechtstreeks in de database aanbrengt. Als je de indextoewijzing wijzigt (cf. records.json ), dan moet je klikken op 'Index verwijderen en opnieuw indexeren'.", "indexInEsDoneError": "Er is een fout inde index. Bekijk de logs voor meer details.", "indexInEsDone": "Het indexeren is gelukt", - "indexCommit": "Wijzigingen in de index bevestigen", - "indexCommit-help": "Alleen gebruiken indien de indexatie taak vastgelopen is.", - "indexCommitError": "Er is een fout opgetreden bij het bevestigen van de index wijzigingen", "ui-moreLikeThisConfig": "Configuratie voor vergelijkbare resultaten", "ui-moreLikeThisConfig-help": "De configuratie moet een more_like_this.like hebben die zal worden ingesteld met de recordtitel om te zoeken naar vergelijkbare records.", "ui-autocompleteConfig": "Auto aanvullen configuratie", diff --git a/web-ui/src/main/resources/catalog/locales/pt-v4.json b/web-ui/src/main/resources/catalog/locales/pt-v4.json index 0cebd4cfc5d..282fa8cfbc5 100644 --- a/web-ui/src/main/resources/catalog/locales/pt-v4.json +++ b/web-ui/src/main/resources/catalog/locales/pt-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/ro-v4.json b/web-ui/src/main/resources/catalog/locales/ro-v4.json index cc5bf3a7857..5c5bdda18c1 100644 --- a/web-ui/src/main/resources/catalog/locales/ro-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ro-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "În timpul reconstrucției indexului, căutarea poate returna rezultate incomplete și operațiunea CSW GetRecords poate fi dezactivată (dacă ați selectat opțiunea din setări). Utilizați această funcție când traficul de catalog este scăzut. Este recomandat să reconstruiți manual indexul de aici atunci când faceți modificări direct în baza de date. Dacă modificați maparea indexului (cf. records.json ), atunci trebuie să faceți clic pe „Ștergeți indexul și reindexați”.", "indexInEsDoneError": "A apărut o eroare cu indexul. Consultați jurnalele pentru detalii", "indexInEsDone": "Operația de indexare a avut succes", - "indexCommit": "Schimbări ale indexului", - "indexCommit-help": "A se utiliza numai dacă sarcina de indexare este suspendată.", - "indexCommitError": "Eroare la efectuarea modificărilor indexului.", "ui-moreLikeThisConfig": "Mai mult ca această configurație", "ui-moreLikeThisConfig-help": "Configurația trebuie să aibă un more_like_this.like care va fi setat cu titlul înregistrării pentru a căuta înregistrări similare.", "ui-autocompleteConfig": "Configurație de completare automată", diff --git a/web-ui/src/main/resources/catalog/locales/ru-v4.json b/web-ui/src/main/resources/catalog/locales/ru-v4.json index 40f202e1ed4..12b381344d0 100644 --- a/web-ui/src/main/resources/catalog/locales/ru-v4.json +++ b/web-ui/src/main/resources/catalog/locales/ru-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/sk-v4.json b/web-ui/src/main/resources/catalog/locales/sk-v4.json index 6348aa506a7..99f078598c5 100644 --- a/web-ui/src/main/resources/catalog/locales/sk-v4.json +++ b/web-ui/src/main/resources/catalog/locales/sk-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/locales/sv-v4.json b/web-ui/src/main/resources/catalog/locales/sv-v4.json index 6182b700118..bb86e19b01d 100644 --- a/web-ui/src/main/resources/catalog/locales/sv-v4.json +++ b/web-ui/src/main/resources/catalog/locales/sv-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Medan index byggs om kan sökningen returnera ofullständiga resultat och CSW GetRecords-funktionen kan inaktiveras (om du valde det alternativet i inställningarna). Använd den här funktionen när katalogtrafiken är låg. Det rekommenderas att bygga om index manuellt härifrån om du gör ändringar direkt i databasen. Om du ändrar indexmappning måste du klicka på 'Ta bort index och indexera om'.", "indexInEsDoneError": "Det finns ett fel med indexet. Se loggarna för mer information", "indexInEsDone": "Indexeringen lyckades", - "indexCommit": "Spara indexändringar", - "indexCommit-help": "Används endast om indexeringen hakat upp sig.", - "indexCommitError": "Fel vid indexändringar.", "ui-moreLikeThisConfig": "Mer som den här konfigurationen", "ui-moreLikeThisConfig-help": "Konfigurationen måste ha en more_like_this.like som kommer att ställas in med posttiteln för att söka efter liknande poster.", "ui-autocompleteConfig": "Konfiguration av autokomplettering", diff --git a/web-ui/src/main/resources/catalog/locales/uk-v4.json b/web-ui/src/main/resources/catalog/locales/uk-v4.json index 0d75716270a..4b99c230161 100644 --- a/web-ui/src/main/resources/catalog/locales/uk-v4.json +++ b/web-ui/src/main/resources/catalog/locales/uk-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "Під час перебудови індексу пошук може видати неповні результати, а операцію CSW GetRecords можна вимкнути (якщо ви вибрали відповідний параметр у налаштуваннях). Використовуйте цю функцію, коли трафік каталогу низький. Рекомендується перебудувати індекс вручну звідси, якщо вносити зміни безпосередньо в базу даних. Якщо ви змінюєте відображення індексу (див. records.json ), то вам потрібно натиснути «Видалити індекс і повторний індекс».", "indexInEsDoneError": "Виникла помилка з індексом. Подробиці дивіться в журналах", "indexInEsDone": "Операція індексації пройшла успішно", - "indexCommit": "Зафіксувати зміни індексу", - "indexCommit-help": "Використовується, лише якщо завдання індексування зависає.", - "indexCommitError": "Помилка під час фіксації змін індексу.", "ui-moreLikeThisConfig": "Більше подібної конфігурації", "ui-moreLikeThisConfig-help": "Конфігурація повинна мати more_like_this.like, який буде встановлено з назвою запису для пошуку подібних записів.", "ui-autocompleteConfig": "Конфігурація автозавершення", diff --git a/web-ui/src/main/resources/catalog/locales/zh-v4.json b/web-ui/src/main/resources/catalog/locales/zh-v4.json index 016f987cb77..1aaea358618 100644 --- a/web-ui/src/main/resources/catalog/locales/zh-v4.json +++ b/web-ui/src/main/resources/catalog/locales/zh-v4.json @@ -165,9 +165,6 @@ "rebuildIndexHelp": "While rebuilding index, search may return incomplete results and the CSW GetRecords operation can be disabled (if you selected the option in the settings). Use this function, when catalog traffic is low. It's recommended to rebuild index manually from here when making changes directly in the database. If you change index mapping (cf. records.json), then you have to click on 'Delete index and reindex'.", "indexInEsDoneError": "There is an error with the index. See the logs for details", "indexInEsDone": "The indexing operation was successfull", - "indexCommit": "Commit index changes", - "indexCommit-help": "To use only if indexing task is hanging.", - "indexCommitError": "Error while committing index changes.", "ui-moreLikeThisConfig": "More like this configuration", "ui-moreLikeThisConfig-help": "Configuration must have a more_like_this.like which will be set with the record title to search for similar records.", "ui-autocompleteConfig": "Autocompletion configuration", diff --git a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html index adc51c0c483..1c863a7ca7f 100644 --- a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html +++ b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html @@ -39,22 +39,6 @@

indexingInProgress

rebuildIndexHelp
-
-
-
- -
-
indexCommit-help
-
-
From 34e1607a7b595d47a8b0050e57eda6f260a4192f Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Tue, 5 Nov 2024 13:34:13 +0100 Subject: [PATCH 08/18] Change how a running index job is determined --- .../org/fao/geonet/kernel/DataManager.java | 5 -- .../kernel/datamanager/IMetadataIndexer.java | 7 --- .../datamanager/base/BaseMetadataIndexer.java | 5 -- .../datamanager/base/BaseMetadataManager.java | 23 +++----- .../geonet/kernel/search/EsSearchManager.java | 5 -- .../index/BatchOpsMetadataReindexer.java | 55 +++++++++++++------ .../fao/geonet/component/csw/GetRecords.java | 4 +- .../java/org/fao/geonet/api/site/SiteApi.java | 12 ++-- .../org/fao/geonet/services/main/Info.java | 4 +- .../catalog/js/admin/AdminToolsController.js | 5 +- 10 files changed, 54 insertions(+), 71 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index b60d660fb6b..4564bbd2994 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -154,11 +154,6 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) metadataIndexer.batchIndexInThreadPool(context, metadataIds); } - @Deprecated - public boolean isIndexing() { - return metadataIndexer.isIndexing(); - } - @Deprecated public void indexMetadata(final List metadataIds) throws Exception { metadataIndexer.indexMetadata(metadataIds); diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java index cc8315f4f87..190f09aca1f 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java @@ -73,13 +73,6 @@ public interface IMetadataIndexer { */ void batchIndexInThreadPool(ServiceContext context, List metadataIds); - /** - * Is the platform currently indexing? - * - * @return - */ - boolean isIndexing(); - /** * Index the list of records passed as parameter in order. * diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index a5965069037..5d35f561893 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -289,11 +289,6 @@ public void batchIndexInThreadPool(ServiceContext context, List metadataIds) executor.shutdown(); } - @Override - public boolean isIndexing() { - return searchManager.isIndexing(); - } - @Override public void indexMetadata(final List metadataIds) throws Exception { try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(metadataIds.size())) { diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index f26a7479277..68680f5f32a 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -180,12 +180,9 @@ public void init(ServiceContext context, Boolean force) throws Exception { /** * Refresh index if needed. Can also be called after GeoNetwork startup in * order to rebuild the lucene index - * t * - * @param force Force reindexing all from scratch - * @param asynchronous **/ - public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolean asynchronous) throws Exception { + public void synchronizeDbWithIndex(ServiceContext context) throws Exception { // get lastchangedate of all metadata in index Map docs = searchManager.getDocsChangeDate(); @@ -226,7 +223,7 @@ public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolea LOGGER_DATA_MANAGER.debug("- idxLastChange: {}", idxLastChange); // date in index contains 't', date in DBMS contains 'T' - if (force || !idxLastChange.equalsIgnoreCase(lastChange)) { + if (!idxLastChange.equalsIgnoreCase(lastChange)) { LOGGER_DATA_MANAGER.debug("- will be indexed"); toIndex.add(id); } @@ -240,18 +237,14 @@ public void synchronizeDbWithIndex(ServiceContext context, Boolean force, Boolea // if anything to index then schedule it to be done after servlet is // up so that any links to local fragments are resolvable - if (toIndex.size() > 0) { - if (asynchronous) { - Set integerList = toIndex.stream().map(Integer::parseInt).collect(Collectors.toSet()); - new BatchOpsMetadataReindexer( - context.getBean(DataManager.class), - integerList).process(settingManager.getSiteId(), false); - } else { - metadataIndexer.batchIndexInThreadPool(context, toIndex); - } + if (!toIndex.isEmpty()) { + Set integerList = toIndex.stream().map(Integer::parseInt).collect(Collectors.toSet()); + new BatchOpsMetadataReindexer( + context.getBean(DataManager.class), + integerList).process(settingManager.getSiteId(), false); } - if (docs.size() > 0) { // anything left? + if (!docs.isEmpty()) { // anything left? LOGGER_DATA_MANAGER.debug("INDEX HAS RECORDS THAT ARE NOT IN DB:"); } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index a1433fbc244..90846839d94 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -904,11 +904,6 @@ public static String analyzeField(String analyzer, fieldValue); } - public boolean isIndexing() { - // TODO this needs a new metric - return false; - } - public boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException { String indexBlockRead = "index.blocks.read_only_allow_delete"; diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index 12a41ab2564..d1f60f5dd54 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -50,7 +50,6 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.MetadataIndexerProcessor; -import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.util.ThreadUtils; import org.fao.geonet.utils.Log; @@ -59,8 +58,10 @@ import org.springframework.jmx.export.annotation.ManagedResource; import javax.management.ObjectName; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -75,11 +76,17 @@ @ManagedResource() public class BatchOpsMetadataReindexer extends MetadataIndexerProcessor implements Runnable { - private static final JmxRemovalListener removalListener = new JmxRemovalListener(); + private static final JmxRemovalListener REMOVAL_LISTENER = new JmxRemovalListener(); private static final Cache PROBE_CACHE = CacheBuilder.newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) - .removalListener(removalListener) + .removalListener(REMOVAL_LISTENER) .build(); + private static final Set> RUNNING_INDEXERS = Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())); + + public static boolean isIndexing() { + RUNNING_INDEXERS.removeIf(CompletableFuture::isDone); + return !RUNNING_INDEXERS.isEmpty(); + } private final Set metadata; private ExecutorService executor = null; @@ -95,7 +102,7 @@ public BatchOpsMetadataReindexer(DataManager dm, Set metadata) { this.metadata = metadata; this.toProcessCount = metadata.size(); exporter = ApplicationContextHolder.get().getBean(MBeanExporter.class); - removalListener.setExporter(exporter); + REMOVAL_LISTENER.setExporter(exporter); } @ManagedAttribute @@ -130,15 +137,9 @@ public String wrapAsyncProcess(String catalogueId, boolean runInCurrentThread) t return processAsync(runInCurrentThread); } - private String processAsync(boolean runInCurrentThread) throws Exception { + private String processAsync(boolean runInCurrentThread) { int threadCount = ThreadUtils.getNumberOfThreads(); - if (runInCurrentThread) { - executor = MoreExecutors.newDirectExecutorService(); - } else { - executor = Executors.newFixedThreadPool(threadCount); - } - int[] ids = metadata.stream().mapToInt(i -> i).toArray(); int perThread; @@ -162,20 +163,38 @@ private String processAsync(boolean runInCurrentThread) throws Exception { index += count; } - List submitList = Lists.newArrayList(); - for (BatchOpsCallable job : jobs) { - CompletableFuture completed = CompletableFuture.runAsync(job, executor); - submitList.add(completed); - } + if (runInCurrentThread) { + allCompleted = new CompletableFuture<>(); + try { + RUNNING_INDEXERS.add(allCompleted); + for (BatchOpsCallable job : jobs) { + job.run(); + } + } + finally { + allCompleted.complete(null); + } + } else { + List> submitList = Lists.newArrayList(); + executor = Executors.newFixedThreadPool(threadCount); - allCompleted = CompletableFuture.allOf(submitList.toArray(new CompletableFuture[submitList.size()])); + for (BatchOpsCallable job : jobs) { + CompletableFuture completed = CompletableFuture.runAsync(job, executor); + submitList.add(completed); + } + + allCompleted = CompletableFuture.allOf(submitList.toArray(new CompletableFuture[0])); + RUNNING_INDEXERS.add(allCompleted); + } allCompleted.thenRun(this); return probeName.toString(); } @Override public void run() { - executor.shutdown(); + if (executor != null) { + executor.shutdown(); + } PROBE_CACHE.cleanUp(); PROBE_CACHE.put(probeName, probeName); } diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java index 5b0fb4e224d..abf88a6e550 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/GetRecords.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.csw.services.getrecords.FieldMapper; import org.fao.geonet.kernel.csw.services.getrecords.SearchController; import org.fao.geonet.kernel.csw.services.getrecords.SortByParser; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.CustomElementSetRepository; @@ -129,10 +130,9 @@ public Element execute(Element request, ServiceContext context) throws CatalogEx // Return exception is indexing. GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME); - DataManager dataManager = gc.getBean(DataManager.class); SettingManager settingsManager = gc.getBean(SettingManager.class); if (!settingsManager.getValueAsBool(SYSTEM_CSW_ENABLEWHENINDEXING) && - dataManager.isIndexing()) { + BatchOpsMetadataReindexer.isIndexing()) { throw new RuntimeException("Catalog is indexing records, retry later."); } diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java index ed3154ea1d7..e024af209fe 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java @@ -61,6 +61,7 @@ import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; import org.fao.geonet.kernel.harvest.HarvestManager; import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -553,7 +554,7 @@ public boolean isIndexing( HttpServletRequest request ) throws Exception { ApiUtils.createServiceContext(request); - return ApplicationContextHolder.get().getBean(DataManager.class).isIndexing(); + return BatchOpsMetadataReindexer.isIndexing(); } @io.swagger.v3.oas.annotations.Operation( @@ -570,10 +571,6 @@ public HttpEntity indexSite( required = false) @RequestParam(required = false, defaultValue = "true") boolean reset, - @Parameter(description = "Asynchronous mode (only on all records. ie. no selection bucket)", - required = false) - @RequestParam(required = false, defaultValue = "false") - boolean asynchronous, @Parameter(description = "Index. By default only remove record index.", required = false) @RequestParam(required = false, defaultValue = "records") @@ -589,8 +586,7 @@ public HttpEntity indexSite( ) throws Exception { ServiceContext context = ApiUtils.createServiceContext(request); EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); - DataManager dataManager = ApplicationContextHolder.get().getBean(DataManager.class); - boolean isIndexing = dataManager.isIndexing(); + boolean isIndexing = BatchOpsMetadataReindexer.isIndexing(); if (isIndexing) { throw new NotAllowedException( @@ -606,7 +602,7 @@ public HttpEntity indexSite( if (StringUtils.isEmpty(bucket)) { BaseMetadataManager metadataManager = ApplicationContextHolder.get().getBean(BaseMetadataManager.class); - metadataManager.synchronizeDbWithIndex(context, false, asynchronous); + metadataManager.synchronizeDbWithIndex(context); } else { searchMan.rebuildIndex(context, false, bucket); } diff --git a/services/src/main/java/org/fao/geonet/services/main/Info.java b/services/src/main/java/org/fao/geonet/services/main/Info.java index 6cf34804bc3..cc591635cf3 100644 --- a/services/src/main/java/org/fao/geonet/services/main/Info.java +++ b/services/src/main/java/org/fao/geonet/services/main/Info.java @@ -34,9 +34,9 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.*; import org.fao.geonet.exceptions.BadParameterEx; -import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.region.RegionsDAO; +import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.lib.Lib; @@ -253,7 +253,7 @@ public Element exec(Element inParams, ServiceContext context) throws Exception { */ private Element getIndex(GeonetContext gc) { Element isIndexing = new Element(INDEX); - isIndexing.setText(Boolean.toString(gc.getBean(DataManager.class).isIndexing())); + isIndexing.setText(Boolean.toString(BatchOpsMetadataReindexer.isIndexing())); return isIndexing; } diff --git a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js index 148b1ccffb9..d8d068bfc93 100644 --- a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js @@ -252,10 +252,7 @@ $scope.rebuildIndex = function (dropFirst, index) { var url = - "../api/site/index?reset=" + - dropFirst + - "&asynchronous=true" + - (index ? "&indices=" + index : ""); + "../api/site/index?reset=" + dropFirst + (index ? "&indices=" + index : ""); return $http.put(url).then( function (response) { checkIsIndexing(); From 83ea4482816b051add1d5774f7734883fb533d6a Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Tue, 5 Nov 2024 15:16:54 +0100 Subject: [PATCH 09/18] Fixed using the wrong map in BatchingIndexSubmittor --- .../kernel/search/submission/BatchingIndexSubmittor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java index 559b405d47d..36a14510019 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java @@ -48,11 +48,11 @@ public void run() { private void sendDocumentsToIndex(Map toIndex) { EsRestClient restClient = searchManager.getClient(); - BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), listOfDocumentsToIndex); + BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), toIndex); CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) .thenAcceptAsync(bulkItemResponses -> { try { - searchManager.handleIndexResponse(bulkItemResponses, listOfDocumentsToIndex); + searchManager.handleIndexResponse(bulkItemResponses, toIndex); } catch (IOException e) { throw new UncheckedIOException(e); } From 66e9c389365a3e3bd2cd291f117a48d21e918533 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Thu, 7 Nov 2024 16:28:16 +0100 Subject: [PATCH 10/18] Fix field updating not refreshing This causes issues with some tests. This issue was already present before the changes --- .../main/java/org/fao/geonet/kernel/search/EsSearchManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 90846839d94..36d1d99a25d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -24,6 +24,7 @@ package org.fao.geonet.kernel.search; import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.Refresh; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; @@ -381,6 +382,7 @@ public BulkResponse updateFields(String id, Map fieldMap, Set b.index(defaultIndex) .operations(bulkOperationList) + .refresh(Refresh.True) ); return client.getClient().bulk(bulkRequest); From 948575b5a16466770fe13ce6ccad23587ebf47ef Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Thu, 7 Nov 2024 17:07:24 +0100 Subject: [PATCH 11/18] Fix UserSelectionsApiTest being too strict about the submittor --- .../org/fao/geonet/api/selections/UserSelectionsApiTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java index b2684f22a92..afa379d5752 100644 --- a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java @@ -33,6 +33,7 @@ import org.fao.geonet.kernel.mef.MEFLibIntegrationTest; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; @@ -244,7 +245,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isNoContent()); - verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), eq(DirectIndexSubmittor.INSTANCE), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(IIndexSubmittor.class), eq(IndexingMode.full)); // Delete this.mockMvc.perform(delete("/srv/api/userselections/" + createdSelection.getId()) From 95e4a5783269f0fe215467adec3e1349a8de85e9 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Mon, 11 Nov 2024 15:03:47 +0100 Subject: [PATCH 12/18] Add support for batch deletion as well --- .../fao/geonet/kernel/IndexMetadataTask.java | 5 +- .../kernel/datamanager/IMetadataManager.java | 4 +- .../datamanager/base/BaseMetadataIndexer.java | 53 +++---- .../datamanager/base/BaseMetadataManager.java | 21 ++- .../draft/DraftMetadataManager.java | 5 +- .../datamanager/draft/DraftMetadataUtils.java | 3 +- .../org/fao/geonet/kernel/mef/Importer.java | 3 +- .../geonet/kernel/search/EsSearchManager.java | 39 +++-- .../geonet/kernel/search/ISearchManager.java | 10 +- .../index/BatchOpsMetadataReindexer.java | 3 +- .../kernel/search/index/IndexingTask.java | 4 +- .../submission/BatchingIndexSubmittor.java | 141 ------------------ .../submission/DirectDeletionSubmittor.java | 27 ++++ .../submission/DirectIndexSubmittor.java | 3 +- .../search/submission/IDeletionSubmittor.java | 10 ++ .../batch/BatchingDeletionSubmittor.java | 60 ++++++++ .../batch/BatchingIndexSubmittor.java | 78 ++++++++++ .../batch/BatchingSubmittorBase.java | 51 +++++++ .../search/submission/batch/StateBase.java | 46 ++++++ .../kernel/DataManagerIntegrationTest.java | 3 +- .../kernel/LocalXLinksUpdateDeleteTest.java | 9 +- .../geonet/kernel/url/UrlAnalyzerTest.java | 3 +- .../fao/geonet/component/csw/Transaction.java | 59 ++++---- .../geonet/kernel/harvest/BaseAligner.java | 2 +- .../harvest/harvester/AbstractHarvester.java | 10 +- .../harvester/arcsde/ArcSDEHarvester.java | 21 +-- .../kernel/harvest/harvester/csw/Aligner.java | 15 +- .../harvest/harvester/geoPREST/Aligner.java | 25 ++-- .../harvest/harvester/geonet/Aligner.java | 31 ++-- .../harvest/harvester/geonet20/Aligner.java | 23 +-- .../LocalFilesystemHarvester.java | 19 ++- .../harvest/harvester/ogcwxs/Harvester.java | 31 ++-- .../harvest/harvester/simpleurl/Aligner.java | 16 +- .../harvest/harvester/thredds/Harvester.java | 54 ++++--- .../harvester/wfsfeatures/Harvester.java | 41 ++--- .../org/fao/geonet/index/es/EsRestClient.java | 71 +++++---- .../metadata/draft/DraftUtilities.java | 3 +- .../api/records/MetadataInsertDeleteApi.java | 5 +- .../api/selections/UserSelectionsApi.java | 2 +- .../cache/FormatterCacheIntegrationTest.java | 3 +- 40 files changed, 615 insertions(+), 397 deletions(-) delete mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java create mode 100644 core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index f74f10be15e..4b1e2c858bd 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -28,15 +28,12 @@ import org.fao.geonet.Util; import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.User; -import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.utils.Log; import org.springframework.transaction.TransactionStatus; -import java.io.IOException; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java index f205284d013..dfcdbb201f5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java @@ -33,6 +33,7 @@ import org.fao.geonet.kernel.EditLib; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.repository.BatchUpdateQuery; import org.fao.geonet.repository.PathSpec; @@ -69,9 +70,10 @@ public interface IMetadataManager { * * @param context * @param metadataId + * @param deletionSubmittor * @throws Exception */ - void deleteMetadata(ServiceContext context, String metadataId) throws Exception; + void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor deletionSubmittor) throws Exception; /** * Delete the record with the id metadataId diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 5d35f561893..830bef0269c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -49,7 +49,8 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexFields; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -71,13 +72,11 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.interceptor.TransactionAspectSupport; -import java.io.IOException; import java.nio.file.Path; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; import static org.fao.geonet.resources.Resources.DEFAULT_LOGO_EXTENSION; @@ -170,29 +169,31 @@ public int batchDeleteMetadataAndUpdateIndex(Specification Integer.toString(input.getId())).collect(Collectors.toList())); // metadataManager.deleteAll(specification); // So delete one by one even if slower - metadataToDelete.forEach(md -> { - try { - // Extract information for RecordDeletedEvent - LinkedHashMap titles = metadataUtils.extractTitles(Integer.toString(md.getId())); - UserSession userSession = ServiceContext.get().getUserSession(); - String xmlBefore = md.getData(); - - store.delResources(ServiceContext.get(), md.getUuid()); - metadataManager.deleteMetadata(ServiceContext.get(), String.valueOf(md.getId())); - - // Trigger RecordDeletedEvent - new RecordDeletedEvent(md.getId(), md.getUuid(), titles, userSession.getUserIdAsInt(), xmlBefore).publish(ApplicationContextHolder.get()); - } catch (Exception e) { - Log.warning(Geonet.DATA_MANAGER, String.format( - - "Error during removal of metadata %s part of batch delete operation. " + - "This error may create a ghost record (ie. not in the index " + - "but still present in the database). " + - "You can reindex the catalogue to see it again. " + - "Error was: %s.", md.getUuid(), e.getMessage())); - e.printStackTrace(); - } - }); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(metadataToDelete.size())) { + metadataToDelete.forEach(md -> { + try { + // Extract information for RecordDeletedEvent + LinkedHashMap titles = metadataUtils.extractTitles(Integer.toString(md.getId())); + UserSession userSession = ServiceContext.get().getUserSession(); + String xmlBefore = md.getData(); + + store.delResources(ServiceContext.get(), md.getUuid()); + metadataManager.deleteMetadata(ServiceContext.get(), String.valueOf(md.getId()), submittor); + + // Trigger RecordDeletedEvent + new RecordDeletedEvent(md.getId(), md.getUuid(), titles, userSession.getUserIdAsInt(), xmlBefore).publish(ApplicationContextHolder.get()); + } catch (Exception e) { + Log.warning(Geonet.DATA_MANAGER, String.format( + + "Error during removal of metadata %s part of batch delete operation. " + + "This error may create a ghost record (ie. not in the index " + + "but still present in the database). " + + "You can reindex the catalogue to see it again. " + + "Error was: %s.", md.getUuid(), e.getMessage())); + e.printStackTrace(); + } + }); + } return metadataToDelete.size(); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 68680f5f32a..6ae3401318f 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -50,8 +50,11 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.kernel.setting.SettingInfo; @@ -249,9 +252,11 @@ public void synchronizeDbWithIndex(ServiceContext context) throws Exception { } // remove from index metadata not in DBMS - for (String id : docs.keySet()) { - getSearchManager().delete(String.format("+id:%s", id)); - LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(docs.keySet().size())) { + for (String id : docs.keySet()) { + getSearchManager().deleteById(id, submittor); + LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); + } } } @@ -330,7 +335,7 @@ private XmlSerializer getXmlSerializer() { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor submittor) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { boolean isMetadata = findOne.getDataInfo().getType() == MetadataType.METADATA; @@ -339,7 +344,7 @@ public void deleteMetadata(ServiceContext context, String metadataId) throws Exc } // --- update search criteria - getSearchManager().delete(String.format("+id:%s", metadataId)); + getSearchManager().deleteById(metadataId, submittor); // _entityManager.flush(); // _entityManager.clear(); } @@ -370,7 +375,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( metadata.getId(), metadata.getUuid(), new LinkedHashMap<>(), context.getUserSession().getUserIdAsInt(), metadata.getData()); - deleteMetadata(context, metadataId); + deleteMetadata(context, metadataId, DirectDeletionSubmittor.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); } @@ -382,7 +387,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit @Override public void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); - getSearchManager().delete(String.format("+id:%s", metadataId)); + getSearchManager().deleteById(metadataId, DirectDeletionSubmittor.INSTANCE); } /** @@ -782,7 +787,7 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context if (indexingMode != IndexingMode.none) { // Delete old record if UUID changed if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) { - getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo)); + getSearchManager().deleteByQuery(String.format("+uuid:\"%s\"", uuidBeforeUfo)); } metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, indexingMode); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java index 07db4799fe2..b7ec7f2ff44 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java @@ -32,6 +32,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -69,7 +70,7 @@ public void init(ServiceContext context, Boolean force) throws Exception { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor submittor) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { @@ -89,7 +90,7 @@ public void deleteMetadata(ServiceContext context, String metadataId) throws Exc } // --- update search criteria - getSearchManager().delete(String.format("+id:%s", metadataId)); + getSearchManager().deleteById(metadataId, submittor); // _entityManager.flush(); // _entityManager.clear(); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index 4c8cd76a1b6..3001465dd3e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -41,6 +41,7 @@ import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -652,7 +653,7 @@ public void cancelEditingSession(ServiceContext context, String id) throws Excep // --- remove metadata xmlSerializer.delete(id, ServiceContext.get()); - searchManager.delete(String.format("+id:%s", id)); + searchManager.deleteById(id, DirectDeletionSubmittor.INSTANCE); // Unset METADATA_EDITING_CREATED_DRAFT flag context.getUserSession().removeProperty(Geonet.Session.METADATA_EDITING_CREATED_DRAFT); diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java index 3f0e94e1cff..7ee7b6e008c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java @@ -45,6 +45,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -572,7 +573,7 @@ public static void importRecord(String uuid, MEFLib.UuidAction uuidAction, List< if (Log.isDebugEnabled(Geonet.MEF)) { Log.debug(Geonet.MEF, "Deleting existing metadata with UUID : " + uuid); } - metadataManager.deleteMetadata(context, dm.getMetadataId(uuid)); + metadataManager.deleteMetadata(context, dm.getMetadataId(uuid), DirectDeletionSubmittor.INSTANCE); metadataManager.flush(); } else { throw new UnAuthorizedException("User has no privilege to replace existing metadata", null); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 36d1d99a25d..f58cc494689 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -28,6 +28,7 @@ import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.ExistsRequest; @@ -54,7 +55,9 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.repository.SourceRepository; @@ -70,7 +73,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -466,6 +468,18 @@ public void handleIndexResponse(BulkResponse bulkResponse, Map d } } + public void handleDeletionResponse(BulkResponse bulkResponse, List documents) { + if (bulkResponse.errors()) { + StringBuilder builder = new StringBuilder(); + for (BulkResponseItem item : bulkResponse.items()) { + if (item.error() != null) { + builder.append("Failed to delete document ").append(item.id()).append(" from index: ").append(item.error()).append("\n"); + } + } + throw new RuntimeException("Some documents could not be deleted from the index!\n" + builder.toString()); + } + } + private void checkIndexResponse(BulkResponse bulkItemResponses, Map documents) throws IOException { if (bulkItemResponses.errors()) { @@ -517,7 +531,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, }); if (!listErrorOfDocumentsToIndex.isEmpty()) { - BulkRequest bulkRequest = client.buildBulkRequest(defaultIndex, listErrorOfDocumentsToIndex); + BulkRequest bulkRequest = client.buildIndexBulkRequest(defaultIndex, listErrorOfDocumentsToIndex); BulkResponse response = client.getClient().bulk(bulkRequest); if (response.errors()) { LOGGER.error("Failed to save error documents {}.", @@ -855,19 +869,24 @@ public ISODate getDocChangeDate(String mdId) throws Exception { } @Override - public void delete(String txt) throws Exception { + public void deleteByQuery(String txt) throws Exception { client.deleteByQuery(defaultIndex, txt); } + @Override + public void deleteById(String txt, IDeletionSubmittor submittor) throws Exception { + submittor.submitToIndex(txt, this); + } + @Override public void delete(List metadataIds) throws Exception { - metadataIds.forEach(metadataId -> { - try { - this.delete(String.format("+id:%d", metadataId)); - } catch (Exception e) { - e.printStackTrace(); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(metadataIds.size())) { + for (Integer id : metadataIds) { + deleteById(String.valueOf(id), submittor); } - }); + } catch (Exception e) { + LOGGER.error("Error while deleting metadata: {}", e.getMessage(), e); + } } public long getNumDocs(String query) throws Exception { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index 9f4a39cca75..bbd79966172 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -28,6 +28,7 @@ import jeeves.server.context.ServiceContext; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.MetadataType; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.jdom.Element; @@ -70,9 +71,14 @@ boolean rebuildIndex(ServiceContext context, ISODate getDocChangeDate(String mdId) throws Exception; /** - * deletes a document. + * deletes a document by a query. */ - void delete(String txt) throws Exception; + void deleteByQuery(String query) throws Exception; + + /** + * deletes a document by its id. + */ + void deleteById(String id, IDeletionSubmittor submittor) throws Exception; /** * deletes a list of documents. diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index d1f60f5dd54..623798dc650 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -45,12 +45,11 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalNotification; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.MoreExecutors; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.MetadataIndexerProcessor; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.util.ThreadUtils; import org.fao.geonet.utils.Log; import org.springframework.jmx.export.MBeanExporter; diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index a5a57aec570..40f747b491c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -29,8 +29,7 @@ import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.utils.Log; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -38,7 +37,6 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.scheduling.quartz.QuartzJobBean; -import java.io.IOException; import java.util.Date; import java.util.Set; diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java deleted file mode 100644 index 36a14510019..00000000000 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/BatchingIndexSubmittor.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.fao.geonet.kernel.search.submission; - -import co.elastic.clients.elasticsearch.core.BulkRequest; -import org.fao.geonet.constants.Geonet; -import org.fao.geonet.index.es.EsRestClient; -import org.fao.geonet.kernel.search.EsSearchManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.lang.ref.Cleaner; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -/** - * An index submitter that batches documents into larger chunks and sends them asynchronously to the index. - * When closing, this submittor sends the remaining item to the index and waits until all elasticsearch requests have been received, - * so after closing there are no pending changes - */ -public class BatchingIndexSubmittor implements AutoCloseable, IIndexSubmittor { - private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.INDEX_ENGINE); - private static final Cleaner CLEANER = Cleaner.create(); - /** - * Maximum number of inflight bulk requests before waiting for the Elasticsearch - */ - private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; - - private static class State implements Runnable { - private final Map listOfDocumentsToIndex = new HashMap<>(); - @SuppressWarnings("unchecked") - private final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; - private int index; - private EsSearchManager searchManager; - private boolean closed = false; - - @Override - public void run() { - if (!closed) { - LOGGER.error("BatchingIndexSubmittor was not closed before it was cleaned! Sending any remaining data"); - } - // Send any remaining pending documents - if (!listOfDocumentsToIndex.isEmpty()) { - sendDocumentsToIndex(listOfDocumentsToIndex); - } - } - - private void sendDocumentsToIndex(Map toIndex) { - EsRestClient restClient = searchManager.getClient(); - BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), toIndex); - CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) - .thenAcceptAsync(bulkItemResponses -> { - try { - searchManager.handleIndexResponse(bulkItemResponses, toIndex); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).exceptionally(e -> { - LOGGER.error( - "An error occurred while indexing {} documents in current indexing list.", - toIndex.size(), e); - return null; - }); - - // Request send, sort into queue - // First, see if the previous future already finished - CompletableFuture previousFuture = inflightFutures[index]; - if (previousFuture != null) { - if (!previousFuture.isDone()) { - // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer - LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); - previousFuture.join(); - } - } - inflightFutures[index] = currentIndexFuture; - index = (index + 1) % inflightFutures.length; - } - } - - private final State state = new State(); - private final Cleaner.Cleanable cleanable; - private final int commitInterval; - - public BatchingIndexSubmittor() { - this.commitInterval = 200; - this.cleanable = CLEANER.register(this, state); - } - - /** - * @param estimatedTotalSize The estimated size of documents to index. Does not need to match the actual amount of submitted documents - */ - public BatchingIndexSubmittor(int estimatedTotalSize) { - if (estimatedTotalSize < 0) { - throw new IllegalArgumentException("estimatedTotalSize must not be negative"); - } - - // Compute an ideal commit interval based on estimated size of elements to index - // Try to strike a balance between - // a) Not making enough bulk requests, thus having to wait a long time at the end for a large chunk => try to make at least 8 requests - int elementsPerBatchRequest = estimatedTotalSize / 8; - // b) Making too many requests, adding unnecessary overhead => set the minimum batch size to 20 - elementsPerBatchRequest = Math.max(20, elementsPerBatchRequest); - // c) Growing the listOfDocumentsToIndex too large => set the maximum batch size to 200 - elementsPerBatchRequest = Math.min(200, elementsPerBatchRequest); - this.commitInterval = elementsPerBatchRequest; - this.cleanable = CLEANER.register(this, state); - } - - @Override - public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { - if (state.closed) { - throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); - } - - state.searchManager = searchManager; - Map listOfDocumentsToIndex = state.listOfDocumentsToIndex; - listOfDocumentsToIndex.put(id, jsonDocument); - if (listOfDocumentsToIndex.size() >= commitInterval) { - Map toIndex = new HashMap<>(listOfDocumentsToIndex); - listOfDocumentsToIndex.clear(); - state.sendDocumentsToIndex(toIndex); - } - } - - @Override - public void close() { - if (this.state.closed) { - throw new IllegalStateException("Attempting to close a already closed " + this.getClass().getSimpleName()); - } - this.state.closed = true; - this.cleanable.clean(); - - // Wait for all remaining documents to be received - for (CompletableFuture inflightFuture : state.inflightFutures) { - if (inflightFuture != null) { - inflightFuture.join(); - } - } - } -} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java new file mode 100644 index 00000000000..b05279d1875 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java @@ -0,0 +1,27 @@ +package org.fao.geonet.kernel.search.submission; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.BulkResponse; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class DirectDeletionSubmittor implements IDeletionSubmittor { + public static final DirectDeletionSubmittor INSTANCE = new DirectDeletionSubmittor(); + + private DirectDeletionSubmittor() {} + + @Override + public void submitToIndex(String id, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + List documents = Collections.singletonList(id); + + BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), documents); + final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); + + searchManager.handleDeletionResponse(bulkItemResponses, documents); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java index 6d628b72cf0..2367cf812bd 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java @@ -22,8 +22,7 @@ public void submitToIndex(String id, String jsonDocument, EsSearchManager search EsRestClient restClient = searchManager.getClient(); Map documents = Collections.singletonMap(id, jsonDocument); - BulkRequest bulkRequest = restClient.buildBulkRequest(searchManager.getDefaultIndex(), documents); - // TODO log any exception + BulkRequest bulkRequest = restClient.buildIndexBulkRequest(searchManager.getDefaultIndex(), documents); final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); searchManager.handleIndexResponse(bulkItemResponses, documents); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java new file mode 100644 index 00000000000..2ea0c86483d --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java @@ -0,0 +1,10 @@ +package org.fao.geonet.kernel.search.submission; + +import org.fao.geonet.kernel.search.EsSearchManager; + +import java.io.IOException; + +public interface IDeletionSubmittor { + + void submitToIndex(String id, EsSearchManager searchManager) throws IOException; +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java new file mode 100644 index 00000000000..c643fd42552 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java @@ -0,0 +1,60 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +public class BatchingDeletionSubmittor extends BatchingSubmittorBase implements IDeletionSubmittor { + + protected static class State extends StateBase { + private final List listOfIDsToDelete = new ArrayList<>(); + + @Override + protected void cleanUp() { + // Send any remaining pending documents + if (!listOfIDsToDelete.isEmpty()) { + deleteDocumentsFromIndex(listOfIDsToDelete); + } + } + + private void deleteDocumentsFromIndex(List toDelete) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), toDelete); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + searchManager.handleDeletionResponse(bulkItemResponses, toDelete); + }); + queueFuture(currentIndexFuture); + } + } + + public BatchingDeletionSubmittor() { + super(new State()); + } + + public BatchingDeletionSubmittor(int estimatedTotalSize) { + super(new State(), estimatedTotalSize); + } + + @Override + public void submitToIndex(String id, EsSearchManager searchManager) throws IOException { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + List listOfIDsToDelete = state.listOfIDsToDelete; + listOfIDsToDelete.add(id); + if (listOfIDsToDelete.size() >= commitInterval) { + List toDelete = new ArrayList<>(listOfIDsToDelete); + listOfIDsToDelete.clear(); + state.deleteDocumentsFromIndex(toDelete); + } + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java new file mode 100644 index 00000000000..45537ca97d4 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java @@ -0,0 +1,78 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import co.elastic.clients.elasticsearch.core.BulkRequest; +import org.fao.geonet.index.es.EsRestClient; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.submission.IIndexSubmittor; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.ref.Cleaner; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * An index submitter that batches documents into larger chunks and sends them asynchronously to the index. + * When closing, this submittor sends the remaining item to the index and waits until all elasticsearch requests have been received, + * so after closing there are no pending changes + */ +public class BatchingIndexSubmittor extends BatchingSubmittorBase implements IIndexSubmittor { + protected static class State extends StateBase { + private final Map listOfDocumentsToIndex = new HashMap<>(); + + @Override + protected void cleanUp() { + // Send any remaining pending documents + if (!listOfDocumentsToIndex.isEmpty()) { + sendDocumentsToIndex(listOfDocumentsToIndex); + } + } + + private void sendDocumentsToIndex(Map toIndex) { + EsRestClient restClient = searchManager.getClient(); + BulkRequest bulkRequest = restClient.buildIndexBulkRequest(searchManager.getDefaultIndex(), toIndex); + CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) + .thenAcceptAsync(bulkItemResponses -> { + try { + searchManager.handleIndexResponse(bulkItemResponses, toIndex); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).exceptionally(e -> { + LOGGER.error( + "An error occurred while indexing {} documents in current indexing list.", + toIndex.size(), e); + return null; + }); + queueFuture(currentIndexFuture); + } + } + + public BatchingIndexSubmittor() { + super(new State()); + } + + /** + * @param estimatedTotalSize The estimated size of documents to index. Does not need to match the actual amount of submitted documents + */ + public BatchingIndexSubmittor(int estimatedTotalSize) { + super(new State(), estimatedTotalSize); + } + + @Override + public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + Map listOfDocumentsToIndex = state.listOfDocumentsToIndex; + listOfDocumentsToIndex.put(id, jsonDocument); + if (listOfDocumentsToIndex.size() >= commitInterval) { + Map toIndex = new HashMap<>(listOfDocumentsToIndex); + listOfDocumentsToIndex.clear(); + state.sendDocumentsToIndex(toIndex); + } + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java new file mode 100644 index 00000000000..cabce9bfb73 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java @@ -0,0 +1,51 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import java.lang.ref.Cleaner; +import java.util.concurrent.CompletableFuture; + +public abstract class BatchingSubmittorBase implements AutoCloseable { + private static final Cleaner CLEANER = Cleaner.create(); + protected final STATE state; + protected final int commitInterval; + private final Cleaner.Cleanable cleanable; + + protected BatchingSubmittorBase(final STATE state) { + this.commitInterval = 200; + this.state = state; + this.cleanable = CLEANER.register(this, state); + } + + protected BatchingSubmittorBase(final STATE state, int estimatedTotalSize) { + if (estimatedTotalSize < 0) { + throw new IllegalArgumentException("estimatedTotalSize must not be negative"); + } + + // Compute an ideal commit interval based on estimated size of elements to index + // Try to strike a balance between + // a) Not making enough bulk requests, thus having to wait a long time at the end for a large chunk => try to make at least 8 requests + int elementsPerBatchRequest = estimatedTotalSize / 8; + // b) Making too many requests, adding unnecessary overhead => set the minimum batch size to 20 + elementsPerBatchRequest = Math.max(20, elementsPerBatchRequest); + // c) Growing the listOfDocumentsToIndex too large => set the maximum batch size to 200 + elementsPerBatchRequest = Math.min(200, elementsPerBatchRequest); + this.commitInterval = elementsPerBatchRequest; + this.state = state; + this.cleanable = CLEANER.register(this, state); + } + + @Override + public void close() { + if (this.state.closed) { + throw new IllegalStateException("Attempting to close a already closed " + this.getClass().getSimpleName()); + } + this.state.closed = true; + this.cleanable.clean(); + + // Wait for all remaining documents to be received + for (CompletableFuture inflightFuture : state.inflightFutures) { + if (inflightFuture != null) { + inflightFuture.join(); + } + } + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java new file mode 100644 index 00000000000..f816fb4a619 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/StateBase.java @@ -0,0 +1,46 @@ +package org.fao.geonet.kernel.search.submission.batch; + +import org.fao.geonet.constants.Geonet; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CompletableFuture; + +public abstract class StateBase implements Runnable { + protected static final Logger LOGGER = LoggerFactory.getLogger(Geonet.INDEX_ENGINE); + /** + * Maximum number of inflight bulk requests before waiting for the Elasticsearch + */ + private static final int MAX_INFLIGHT_INDEX_REQUESTS = 4; + @SuppressWarnings("unchecked") + protected final CompletableFuture[] inflightFutures = new CompletableFuture[MAX_INFLIGHT_INDEX_REQUESTS]; + protected int index = 0; + protected boolean closed = false; + protected EsSearchManager searchManager; + + @Override + public final void run() { + if (!closed) { + LOGGER.error("BatchingIndexSubmittor was not closed before it was cleaned! Sending any remaining data"); + } + cleanUp(); + } + + protected void queueFuture(CompletableFuture newFuture) { + // Request send, sort into queue + // First, see if the previous future already finished + CompletableFuture previousFuture = inflightFutures[index]; + if (previousFuture != null) { + if (!previousFuture.isDone()) { + // Normally, the ES should be able to keep up. If it does not, just wait until there is some space in the ring buffer + LOGGER.info("Waiting for elasticsearch to process pending bulk requests..."); + previousFuture.join(); + } + } + inflightFutures[index] = newFuture; + index = (index + 1) % inflightFutures.length; + } + + protected abstract void cleanUp(); +} diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index fa7b010b9b1..04a0cbc89da 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -41,6 +41,7 @@ import org.fao.geonet.domain.User; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -78,7 +79,7 @@ public void testDeleteMetadata() throws Exception { assertEquals(count + 1, metadataRepository.count()); - metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId)); + metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId), DirectDeletionSubmittor.INSTANCE); assertEquals(count, metadataRepository.count()); } diff --git a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java index e67834f85ef..048b3d5cf3c 100644 --- a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java @@ -5,6 +5,7 @@ import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Xml; @@ -74,8 +75,8 @@ public void deleteAllowedWhenRefNotExists() throws Exception { AbstractMetadata contactMetadata = insertContact(); AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId())); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId())); + metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId()), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } @@ -87,7 +88,7 @@ public void deleteHasToBeForbiddenWhenRefExistsAndSettingsSaySo() throws Excepti try { metadataManager.deleteMetadata(serviceContext, - Integer.toString(contactMetadata.getId())); + Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); } catch (Exception e) { } @@ -100,7 +101,7 @@ public void deleteHasToBeAllowedWhenRefExistsAndSettingsSaySo() throws Exception AbstractMetadata contactMetadata = insertContact(); insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId())); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } diff --git a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java index f43915553b0..8f6926eec72 100644 --- a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java @@ -11,6 +11,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.LinkRepository; @@ -188,7 +189,7 @@ public void deleteMetadaLinkWhenMdDisappeared() throws Exception { SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository(MetadataLink.class, entityManager); List metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); - dataManager.deleteMetadata(context, md.getId() + ""); + dataManager.deleteMetadata(context, md.getId() + "", DirectDeletionSubmittor.INSTANCE); linkRepository.findAll().stream().forEach(toTest::purgeMetataLink); diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java index 4d7a445413d..4452031207f 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java @@ -51,6 +51,7 @@ import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Log; @@ -526,38 +527,40 @@ private int deleteTransaction(Element request, ServiceContext context) throws Ex return deleted; // delete all matching records - while (i.hasNext()) { - Element result = i.next(); - String uuid = result.getChildText("identifier", Csw.NAMESPACE_DC); - String id = dataMan.getMetadataId(uuid); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(results.size())) { + while (i.hasNext()) { + Element result = i.next(); + String uuid = result.getChildText("identifier", Csw.NAMESPACE_DC); + String id = dataMan.getMetadataId(uuid); - if (id == null) { - return deleted; - } + if (id == null) { + return deleted; + } - if (!dataMan.getAccessManager().canEdit(context, id)) { - throw new NoApplicableCodeEx("User not allowed to delete metadata : " + id); - } - AbstractMetadata metadata = metadataUtils.findOneByUuid(uuid); - LinkedHashMap titles = new LinkedHashMap<>(); - try { - titles = metadataUtils.extractTitles(Integer.toString(metadata.getId())); - } catch (Exception e) { - Log.warning(Geonet.DATA_MANAGER, - String.format( - "Error while extracting title for the metadata %d " + - "while creating delete event. Error is %s.", - metadata.getId(), e.getMessage())); - } - RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( - metadata.getId(), metadata.getUuid(), titles, - context.getUserSession().getUserIdAsInt(), - metadata.getData()); + if (!dataMan.getAccessManager().canEdit(context, id)) { + throw new NoApplicableCodeEx("User not allowed to delete metadata : " + id); + } + AbstractMetadata metadata = metadataUtils.findOneByUuid(uuid); + LinkedHashMap titles = new LinkedHashMap<>(); + try { + titles = metadataUtils.extractTitles(Integer.toString(metadata.getId())); + } catch (Exception e) { + Log.warning(Geonet.DATA_MANAGER, + String.format( + "Error while extracting title for the metadata %d " + + "while creating delete event. Error is %s.", + metadata.getId(), e.getMessage())); + } + RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( + metadata.getId(), metadata.getUuid(), titles, + context.getUserSession().getUserIdAsInt(), + metadata.getData()); - metadataManager.deleteMetadata(context, id); - recordDeletedEvent.publish(ApplicationContextHolder.get()); + metadataManager.deleteMetadata(context, id, submittor); + recordDeletedEvent.publish(ApplicationContextHolder.get()); - deleted++; + deleted++; + } } return deleted; diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java index 09ad3b5af6e..99c918fbb9f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java @@ -35,7 +35,7 @@ import org.fao.geonet.kernel.harvest.harvester.CategoryMapper; import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.Privileges; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataCategoryRepository; diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java index 2398aa96c10..d70e366e0f6 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java @@ -49,6 +49,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.harvest.Common.OperResult; import org.fao.geonet.kernel.harvest.Common.Status; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.HarvesterSettingsManager; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -278,9 +279,12 @@ public void destroy() throws Exception { final Specification ownedByHarvester = Specification.where(MetadataSpecs.hasHarvesterUuid(getParams().getUuid())); Set sources = new HashSet<>(); - for (Integer metadataId : metadataRepository.findAllIdsBy(ownedByHarvester)) { - sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId()); - metadataManager.deleteMetadata(context, "" + metadataId); + List allIdsBy = metadataRepository.findAllIdsBy(ownedByHarvester); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(allIdsBy.size())) { + for (Integer metadataId : allIdsBy) { + sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId()); + metadataManager.deleteMetadata(context, "" + metadataId, submittor); + } } // Remove all sources related to the harvestUuid if they are not linked to any record anymore diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java index 5e847b8a718..00e043e1a6a 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java @@ -28,7 +28,6 @@ import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.Logger; import org.fao.geonet.arcgis.ArcSDEConnection; -import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Metadata; @@ -42,8 +41,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -308,15 +307,17 @@ private void align(Map metadataList) throws Exception { // Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - for (Integer existingId : existingMetadata) { + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (Integer existingId : existingMetadata) { - if (cancelMonitor.get()) { - return; - } - if (!idsResultHs.contains(existingId)) { - log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString()); - result.locallyRemoved++; + if (cancelMonitor.get()) { + return; + } + if (!idsResultHs.contains(existingId)) { + log.debug(" Removing: " + existingId); + metadataManager.deleteMetadata(context, existingId.toString(), submittor); + result.locallyRemoved++; + } } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 6408ba24fdd..3bdeb588bff 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -57,6 +57,7 @@ import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Xml; import org.jdom.Element; @@ -256,12 +257,14 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - for (String uuid : localUuids.getUUIDs()) { - if (!records.contains(uuid)) { - String id = localUuids.getID(uuid); - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); - result.locallyRemoved++; + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + if (!records.contains(uuid)) { + String id = localUuids.getID(uuid); + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submittor); + result.locallyRemoved++; + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index a82e1ccfd04..df979b0c5b0 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -51,6 +51,7 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.utils.GeonetHttpRequestFactory; import org.fao.geonet.utils.Xml; @@ -105,21 +106,23 @@ public HarvestResult align(Set records, List errors) t //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return result; - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return result; + } - if (!exists(records, uuid)) { - String id = localUuids.getID(uuid); + if (!exists(records, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submittor); - metadataManager.flush(); + metadataManager.flush(); - result.locallyRemoved++; + result.locallyRemoved++; + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java index 7fb63ce035e..292af9b68ee 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.*; import org.fao.geonet.kernel.mef.*; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.GroupRepository; @@ -150,24 +151,26 @@ public HarvestResult align(SortedSet records, List err //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return this.result; + } - try { - if (!exists(records, uuid)) { - String id = localUuids.getID(uuid); + try { + if (!exists(records, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id:" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id:" + id); + metadataManager.deleteMetadata(context, id, submittor); - result.locallyRemoved++; + result.locallyRemoved++; + } + } catch (Exception t) { + log.error("Couldn't remove metadata with uuid " + uuid); + log.error(t); + result.unchangedMetadata++; } - } catch (Exception t) { - log.error("Couldn't remove metadata with uuid " + uuid); - log.error(t); - result.unchangedMetadata++; } } //--- insert/update new metadata diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java index c1af6cdbcae..d8ced85f476 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java @@ -43,6 +43,7 @@ import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.repository.specification.MetadataCategorySpecs; import org.fao.geonet.utils.XmlRequest; @@ -105,19 +106,21 @@ public HarvestResult align(Element result, String siteId) throws Exception { //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return this.result; + } - if (!exists(mdList, uuid)) { - String id = localUuids.getID(uuid); + if (!exists(mdList, uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id=" + id); - metadataManager.deleteMetadata(context, id); + if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id=" + id); + metadataManager.deleteMetadata(context, id, submittor); - metadataManager.flush(); - this.result.locallyRemoved++; + metadataManager.flush(); + this.result.locallyRemoved++; + } } } //----------------------------------------------------------------------- diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java index 1b723a12d06..296f61ade62 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java @@ -45,6 +45,7 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.specification.MetadataSpecs; import org.fao.geonet.utils.IO; @@ -113,15 +114,17 @@ private HarvestResult align(Path root) throws Exception { "from the same source if they " + " were not in this harvesting result..."); List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - for (Integer existingId : existingMetadata) { + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (Integer existingId : existingMetadata) { - if (cancelMonitor.get()) { - return this.result; - } - if (!idsResultHs.contains(existingId)) { - log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString()); - result.locallyRemoved++; + if (cancelMonitor.get()) { + return this.result; + } + if (!idsResultHs.contains(existingId)) { + log.debug(" Removing: " + existingId); + metadataManager.deleteMetadata(context, existingId.toString(), submittor); + result.locallyRemoved++; + } } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/Harvester.java index 0a92c4c31fa..109ee84f1ac 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/ogcwxs/Harvester.java @@ -47,6 +47,7 @@ import org.fao.geonet.kernel.schema.ISOPlugin; import org.fao.geonet.kernel.schema.SchemaPlugin; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataCategoryRepository; @@ -248,25 +249,27 @@ public String apply(String uuid) { //----------------------------------------------------------------------- //--- remove old metadata - for (String uuid : localUuids.getUUIDs()) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + if (cancelMonitor.get()) { + return this.result; + } - //If it was not on the uuids added: - if (!uuids.contains(uuid)) { - String id = localUuids.getID(uuid); + //If it was not on the uuids added: + if (!uuids.contains(uuid)) { + String id = localUuids.getID(uuid); - if (log.isDebugEnabled()) - log.debug(" - Removing old metadata before update with id: " + id); + if (log.isDebugEnabled()) + log.debug(" - Removing old metadata before update with id: " + id); - //--- remove the metadata directory including the public and private directories. - store.delResources(context, uuid); + //--- remove the metadata directory including the public and private directories. + store.delResources(context, uuid); - // Remove metadata - metadataManager.deleteMetadata(context, id); + // Remove metadata + metadataManager.deleteMetadata(context, id, submittor); - result.locallyRemoved++; + result.locallyRemoved++; + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java index 72b5ed23299..8464f1accbb 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.repository.OperationAllowedRepository; import org.jdom.Element; @@ -178,12 +179,15 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - for (String uuid : localUuids.getUUIDs()) { - if (!records.contains(uuid)) { - String id = localUuids.getID(uuid); - log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id); - result.locallyRemoved ++; + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + + for (String uuid : localUuids.getUUIDs()) { + if (!records.contains(uuid)) { + String id = localUuids.getID(uuid); + log.debug(" - Removing old metadata with local id:" + id); + metadataManager.deleteMetadata(context, id, submittor); + result.locallyRemoved++; + } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java index ea60589cbe9..937dded6297 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java @@ -57,6 +57,7 @@ import org.fao.geonet.kernel.harvest.harvester.UriMapper; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.util.Sha1Encoder; @@ -289,27 +290,30 @@ public HarvestResult harvest(Logger log) throws Exception { harvestCatalog(xml); //--- Remove previously harvested metadata for uris that no longer exist on the remote site - for (String localUri : localUris.getUris()) { - if (cancelMonitor.get()) { - return this.result; - } - - if (!harvestUris.contains(localUri)) { - for (RecordInfo record : localUris.getRecords(localUri)) { - if (cancelMonitor.get()) { - return this.result; - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { - if (log.isDebugEnabled()) - log.debug(" - Removing deleted metadata with id: " + record.id); - mdManager.deleteMetadata(context, record.id); + for (String localUri : localUris.getUris()) { + if (cancelMonitor.get()) { + return this.result; + } - if (record.isTemplate.equals("s")) { - //--- Uncache xlinks if a subtemplate - Processor.uncacheXLinkUri(metadataGetService + record.uuid); - result.subtemplatesRemoved++; - } else { - result.locallyRemoved++; + if (!harvestUris.contains(localUri)) { + for (RecordInfo record : localUris.getRecords(localUri)) { + if (cancelMonitor.get()) { + return this.result; + } + + if (log.isDebugEnabled()) + log.debug(" - Removing deleted metadata with id: " + record.id); + mdManager.deleteMetadata(context, record.id, submittor); + + if (record.isTemplate.equals("s")) { + //--- Uncache xlinks if a subtemplate + Processor.uncacheXLinkUri(metadataGetService + record.uuid); + result.subtemplatesRemoved++; + } else { + result.locallyRemoved++; + } } } } @@ -785,12 +789,14 @@ private void deleteExistingMetadata(String uri) throws Exception { if (localRecords == null) return; - for (RecordInfo record : localRecords) { - mdManager.deleteMetadata(context, record.id); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(localRecords.size())) { + for (RecordInfo record : localRecords) { + mdManager.deleteMetadata(context, record.id, submittor); - if (record.isTemplate.equals("s")) { - //--- Uncache xlinks if a subtemplate - Processor.uncacheXLinkUri(metadataGetService + record.uuid); + if (record.isTemplate.equals("s")) { + //--- Uncache xlinks if a subtemplate + Processor.uncacheXLinkUri(metadataGetService + record.uuid); + } } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java index 30bfb785647..4ab3ce8d8b1 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java @@ -44,6 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.FragmentParams; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.HarvestSummary; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.lib.Lib; import org.fao.geonet.utils.*; @@ -334,29 +335,31 @@ public void deleteOrphanedMetadata(Set updatedMetadata) throws Exception if (log.isDebugEnabled()) log.debug(" - Removing orphaned metadata records and fragments after update"); - for (String uuid : localUuids.getUUIDs()) { - try { - String isTemplate = localUuids.getTemplate(uuid); - if (isTemplate.equals("s")) { - Processor.uncacheXLinkUri(metadataGetService + uuid); - } + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String uuid : localUuids.getUUIDs()) { + try { + String isTemplate = localUuids.getTemplate(uuid); + if (isTemplate.equals("s")) { + Processor.uncacheXLinkUri(metadataGetService + uuid); + } - if (!updatedMetadata.contains(uuid)) { - String id = localUuids.getID(uuid); - metadataManager.deleteMetadata(context, id); + if (!updatedMetadata.contains(uuid)) { + String id = localUuids.getID(uuid); + metadataManager.deleteMetadata(context, id, submittor); - if (isTemplate.equals("s")) { - result.subtemplatesRemoved++; - } else { - result.locallyRemoved++; + if (isTemplate.equals("s")) { + result.subtemplatesRemoved++; + } else { + result.locallyRemoved++; + } } + } catch (CacheException e) { + HarvestError error = new HarvestError(context, e); + this.errors.add(error); + } catch (Exception e) { + HarvestError error = new HarvestError(context, e); + this.errors.add(error); } - } catch (CacheException e) { - HarvestError error = new HarvestError(context, e); - this.errors.add(error); - } catch (Exception e) { - HarvestError error = new HarvestError(context, e); - this.errors.add(error); } } diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 1f1b6be2507..bac90b0ebb6 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -207,10 +207,14 @@ public EsRestClient setPassword(String password) { return this; } - public BulkRequest buildBulkRequest(String index, Map docs) { + private void checkActivated() { if (!activated) { throw new IllegalStateException("Index not yet activated."); } + } + + public BulkRequest buildIndexBulkRequest(String index, Map docs) { + checkActivated(); BulkRequest.Builder requestBuilder = new BulkRequest.Builder() .index(index) @@ -231,6 +235,43 @@ public BulkRequest buildBulkRequest(String index, Map docs) { return requestBuilder.build(); } + public BulkRequest buildDeleteBulkRequest(String index, List deletionIds) { + checkActivated(); + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() + .index(index) + .refresh(Refresh.True); + + for (String deletionQuery : deletionIds) { + requestBuilder.operations(op -> op.delete(del -> del.index(index) + .id(deletionQuery))); + } + + return requestBuilder.build(); + } + + public void deleteByQuery(String index, String query) throws IOException { + checkActivated(); + + DeleteByQueryRequest request = DeleteByQueryRequest.of( + b -> b.index(index) + .q(query) + .refresh(true)); + + final DeleteByQueryResponse deleteByQueryResponse = + client.deleteByQuery(request); + + + if (!deleteByQueryResponse.failures().isEmpty()) { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + + throw new IOException(String.format( + "Error during removal. Errors are '%s'.", stringBuilder.toString() + )); + } + } + /** * Query using Lucene query syntax. @@ -335,34 +376,6 @@ public SearchResponse query(String index, Query.Builder queryBuilder, Query.Buil } } - - public String deleteByQuery(String index, String query) throws Exception { - if (!activated) { - return ""; - } - - DeleteByQueryRequest request = DeleteByQueryRequest.of( - b -> b.index(new ArrayList<>(Arrays.asList(index))) - .q(query) - .refresh(true)); - - final DeleteByQueryResponse deleteByQueryResponse = - client.deleteByQuery(request); - - - if (deleteByQueryResponse.deleted() >= 0) { - return String.format("Record removed. %s.", deleteByQueryResponse.deleted()); - } else { - StringBuilder stringBuilder = new StringBuilder(); - - deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); - - throw new IOException(String.format( - "Error during removal. Errors are '%s'.", stringBuilder.toString() - )); - } - } - /** * Get the complete document from the index. * @param id For record index, use UUID. diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java index ce418b4062f..be66a714d01 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java @@ -10,6 +10,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; import org.fao.geonet.repository.specification.MetadataValidationSpecs; @@ -186,7 +187,7 @@ public void removeDraft(MetadataDraft draft) { // --- remove metadata xmlSerializer.delete(String.valueOf(id), ServiceContext.get()); - searchManager.delete(String.format("+id:%d", id )); + searchManager.deleteById(String.valueOf(id), DirectDeletionSubmittor.INSTANCE); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Couldn't cleanup draft " + draft, e); } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index cf78db08ec3..fef84962311 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -80,6 +80,7 @@ import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -242,7 +243,7 @@ public void deleteRecord( store.delResources(context, metadata.getUuid(), approved); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, metadata.getId() + ""); - metadataManager.deleteMetadata(context, metadata.getId() + ""); + metadataManager.deleteMetadata(context, metadata.getId() + "", DirectDeletionSubmittor.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); } @@ -301,7 +302,7 @@ public SimpleMetadataProcessingReport deleteRecords( store.delResources(context, metadata.getUuid()); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, String.valueOf(metadata.getId())); - metadataManager.deleteMetadata(context, String.valueOf(metadata.getId())); + metadataManager.deleteMetadata(context, String.valueOf(metadata.getId()), DirectDeletionSubmittor.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); report.incrementProcessedRecords(); diff --git a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java index cad896ad665..d2ade99f3bf 100644 --- a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java +++ b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java @@ -37,7 +37,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.SelectionRepository; diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java index 387a1886882..aba2e64c270 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java @@ -32,6 +32,7 @@ import org.fao.geonet.domain.ReservedOperation; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest.ImportMetadata; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.OperationAllowedSpecs; import org.fao.geonet.services.AbstractServiceIntegrationTest; @@ -134,7 +135,7 @@ public void testUpdatesAfterMetadataDeleted() throws Exception { final Key key = new Key(Integer.parseInt(metadataId), "eng", FormatType.html, "full_view", true, FormatterWidth._100); formatterCache.get(key, new ChangeDateValidator(changeDate), new TestLoader("result", changeDate, true), true); - metadataManager.deleteMetadata(createServiceContext(), metadataId); + metadataManager.deleteMetadata(createServiceContext(), metadataId, DirectDeletionSubmittor.INSTANCE); entityManager.flush(); assertNull(formatterCache.getPublished(key)); From 827539e5c248afb8d02601b735078d4d277d3508 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Mon, 11 Nov 2024 15:42:07 +0100 Subject: [PATCH 13/18] Fix batch deletion --- .../datamanager/base/BaseMetadataManager.java | 21 ++++++++---------- .../draft/DraftMetadataManager.java | 8 +++---- .../datamanager/draft/DraftMetadataUtils.java | 2 +- .../geonet/kernel/search/EsSearchManager.java | 16 ++------------ .../geonet/kernel/search/ISearchManager.java | 9 ++------ .../submission/DirectDeletionSubmittor.java | 4 ++-- .../search/submission/IDeletionSubmittor.java | 2 +- .../batch/BatchingDeletionSubmittor.java | 22 +++++++++---------- .../kernel/DataManagerIntegrationTest.java | 2 ++ .../org/fao/geonet/index/es/EsRestClient.java | 7 +++--- .../metadata/draft/DraftUtilities.java | 2 +- 11 files changed, 37 insertions(+), 58 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 6ae3401318f..83984d3bbf8 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -252,11 +252,9 @@ public void synchronizeDbWithIndex(ServiceContext context) throws Exception { } // remove from index metadata not in DBMS - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(docs.keySet().size())) { - for (String id : docs.keySet()) { - getSearchManager().deleteById(id, submittor); - LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); - } + for (String id : docs.keySet()) { + getSearchManager().deleteByQuery(String.format("+id:%s", id)); + LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); } } @@ -341,12 +339,11 @@ public void deleteMetadata(ServiceContext context, String metadataId, IDeletionS boolean isMetadata = findOne.getDataInfo().getType() == MetadataType.METADATA; deleteMetadataFromDB(context, metadataId); + // --- update search criteria + getSearchManager().deleteByUuid(findOne.getUuid(), submittor); + } else { + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); } - - // --- update search criteria - getSearchManager().deleteById(metadataId, submittor); - // _entityManager.flush(); - // _entityManager.clear(); } /** @@ -387,7 +384,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit @Override public void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); - getSearchManager().deleteById(metadataId, DirectDeletionSubmittor.INSTANCE); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); } /** @@ -787,7 +784,7 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context if (indexingMode != IndexingMode.none) { // Delete old record if UUID changed if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) { - getSearchManager().deleteByQuery(String.format("+uuid:\"%s\"", uuidBeforeUfo)); + getSearchManager().deleteByUuid(uuidBeforeUfo, DirectDeletionSubmittor.INSTANCE); } metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, indexingMode); } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java index b7ec7f2ff44..0cf546d36d0 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java @@ -87,12 +87,10 @@ public void deleteMetadata(ServiceContext context, String metadataId, IDeletionS } deleteMetadataFromDB(context, metadataId); + getSearchManager().deleteByUuid(findOne.getUuid(), submittor); + } else { + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); } - - // --- update search criteria - getSearchManager().deleteById(metadataId, submittor); - // _entityManager.flush(); - // _entityManager.clear(); } /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index 3001465dd3e..a3ec98a249d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -653,7 +653,7 @@ public void cancelEditingSession(ServiceContext context, String id) throws Excep // --- remove metadata xmlSerializer.delete(id, ServiceContext.get()); - searchManager.deleteById(id, DirectDeletionSubmittor.INSTANCE); + searchManager.deleteByQuery(String.format("+id:%s", id)); // Unset METADATA_EDITING_CREATED_DRAFT flag context.getUserSession().removeProperty(Geonet.Session.METADATA_EDITING_CREATED_DRAFT); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index f58cc494689..6cbd9512d0d 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -56,7 +56,6 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; import org.fao.geonet.kernel.setting.SettingInfo; @@ -874,19 +873,8 @@ public void deleteByQuery(String txt) throws Exception { } @Override - public void deleteById(String txt, IDeletionSubmittor submittor) throws Exception { - submittor.submitToIndex(txt, this); - } - - @Override - public void delete(List metadataIds) throws Exception { - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(metadataIds.size())) { - for (Integer id : metadataIds) { - deleteById(String.valueOf(id), submittor); - } - } catch (Exception e) { - LOGGER.error("Error while deleting metadata: {}", e.getMessage(), e); - } + public void deleteByUuid(String uuid, IDeletionSubmittor submittor) throws Exception { + submittor.submitToIndex(uuid, this); } public long getNumDocs(String query) throws Exception { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index bbd79966172..324daf51749 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -76,14 +76,9 @@ boolean rebuildIndex(ServiceContext context, void deleteByQuery(String query) throws Exception; /** - * deletes a document by its id. + * deletes a document by its uuid. */ - void deleteById(String id, IDeletionSubmittor submittor) throws Exception; - - /** - * deletes a list of documents. - */ - void delete(List metadataIds) throws Exception; + void deleteByUuid(String uuid, IDeletionSubmittor submittor) throws Exception; boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java index b05279d1875..eb1c1546dc5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java @@ -15,9 +15,9 @@ public class DirectDeletionSubmittor implements IDeletionSubmittor { private DirectDeletionSubmittor() {} @Override - public void submitToIndex(String id, EsSearchManager searchManager) throws IOException { + public void submitToIndex(String uuid, EsSearchManager searchManager) throws IOException { EsRestClient restClient = searchManager.getClient(); - List documents = Collections.singletonList(id); + List documents = Collections.singletonList(uuid); BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), documents); final BulkResponse bulkItemResponses = restClient.getClient().bulk(bulkRequest); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java index 2ea0c86483d..30384dabff2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java @@ -6,5 +6,5 @@ public interface IDeletionSubmittor { - void submitToIndex(String id, EsSearchManager searchManager) throws IOException; + void submitToIndex(String uuid, EsSearchManager searchManager) throws IOException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java index c643fd42552..10387ef24f6 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java @@ -13,13 +13,13 @@ public class BatchingDeletionSubmittor extends BatchingSubmittorBase implements IDeletionSubmittor { protected static class State extends StateBase { - private final List listOfIDsToDelete = new ArrayList<>(); + private final List listOfUUIDsToDelete = new ArrayList<>(); @Override protected void cleanUp() { // Send any remaining pending documents - if (!listOfIDsToDelete.isEmpty()) { - deleteDocumentsFromIndex(listOfIDsToDelete); + if (!listOfUUIDsToDelete.isEmpty()) { + deleteDocumentsFromIndex(listOfUUIDsToDelete); } } @@ -27,9 +27,7 @@ private void deleteDocumentsFromIndex(List toDelete) { EsRestClient restClient = searchManager.getClient(); BulkRequest bulkRequest = restClient.buildDeleteBulkRequest(searchManager.getDefaultIndex(), toDelete); CompletableFuture currentIndexFuture = restClient.getAsyncClient().bulk(bulkRequest) - .thenAcceptAsync(bulkItemResponses -> { - searchManager.handleDeletionResponse(bulkItemResponses, toDelete); - }); + .thenAcceptAsync(bulkItemResponses -> searchManager.handleDeletionResponse(bulkItemResponses, toDelete)); queueFuture(currentIndexFuture); } } @@ -43,17 +41,17 @@ public BatchingDeletionSubmittor(int estimatedTotalSize) { } @Override - public void submitToIndex(String id, EsSearchManager searchManager) throws IOException { + public void submitToIndex(String uuid, EsSearchManager searchManager) { if (state.closed) { throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); } state.searchManager = searchManager; - List listOfIDsToDelete = state.listOfIDsToDelete; - listOfIDsToDelete.add(id); - if (listOfIDsToDelete.size() >= commitInterval) { - List toDelete = new ArrayList<>(listOfIDsToDelete); - listOfIDsToDelete.clear(); + List listOfUUIDsToDelete = state.listOfUUIDsToDelete; + listOfUUIDsToDelete.add(uuid); + if (listOfUUIDsToDelete.size() >= commitInterval) { + List toDelete = new ArrayList<>(listOfUUIDsToDelete); + listOfUUIDsToDelete.clear(); state.deleteDocumentsFromIndex(toDelete); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index 04a0cbc89da..745fa29e18e 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -184,6 +184,8 @@ public void testDeleteBatchMetadata() throws Exception { dataManager.batchDeleteMetadataAndUpdateIndex(spec); + Thread.sleep(500); + assertEquals(0, metadataRepository.findAll(spec).size()); assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size()); } diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index bac90b0ebb6..919deac8438 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -235,15 +235,16 @@ public BulkRequest buildIndexBulkRequest(String index, Map docs) return requestBuilder.build(); } - public BulkRequest buildDeleteBulkRequest(String index, List deletionIds) { + public BulkRequest buildDeleteBulkRequest(String index, List deletionUUIDs) { checkActivated(); + BulkRequest.Builder requestBuilder = new BulkRequest.Builder() .index(index) .refresh(Refresh.True); - for (String deletionQuery : deletionIds) { + for (String uuid : deletionUUIDs) { requestBuilder.operations(op -> op.delete(del -> del.index(index) - .id(deletionQuery))); + .id(uuid))); } return requestBuilder.build(); diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java index be66a714d01..68bc35e6bde 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java @@ -187,7 +187,7 @@ public void removeDraft(MetadataDraft draft) { // --- remove metadata xmlSerializer.delete(String.valueOf(id), ServiceContext.get()); - searchManager.deleteById(String.valueOf(id), DirectDeletionSubmittor.INSTANCE); + searchManager.deleteByUuid(draft.getUuid(), DirectDeletionSubmittor.INSTANCE); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Couldn't cleanup draft " + draft, e); } From 61fb7adc3bebba5456bf76166089f1c535a08474 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Mon, 11 Nov 2024 16:58:20 +0100 Subject: [PATCH 14/18] Allow delete by query to be "batched" by running them async --- .../datamanager/base/BaseMetadataManager.java | 12 ++++++---- .../draft/DraftMetadataManager.java | 2 +- .../datamanager/draft/DraftMetadataUtils.java | 2 +- .../geonet/kernel/search/EsSearchManager.java | 21 ++++++++++++---- .../geonet/kernel/search/ISearchManager.java | 2 +- .../submission/DirectDeletionSubmittor.java | 14 ++++++++++- .../search/submission/IDeletionSubmittor.java | 4 +++- .../batch/BatchingDeletionSubmittor.java | 23 +++++++++++++++++- .../org/fao/geonet/index/es/EsRestClient.java | 18 ++------------ .../worker/EsWFSFeatureIndexer.java | 24 ++++++++++++++----- 10 files changed, 85 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 83984d3bbf8..055111d8627 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -252,9 +252,11 @@ public void synchronizeDbWithIndex(ServiceContext context) throws Exception { } // remove from index metadata not in DBMS - for (String id : docs.keySet()) { - getSearchManager().deleteByQuery(String.format("+id:%s", id)); - LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); + try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + for (String id : docs.keySet()) { + getSearchManager().deleteByQuery(String.format("+id:%s", id), submittor); + LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); + } } } @@ -342,7 +344,7 @@ public void deleteMetadata(ServiceContext context, String metadataId, IDeletionS // --- update search criteria getSearchManager().deleteByUuid(findOne.getUuid(), submittor); } else { - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submittor); } } @@ -384,7 +386,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit @Override public void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), DirectDeletionSubmittor.INSTANCE); } /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java index 0cf546d36d0..54ad4f4f50e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java @@ -89,7 +89,7 @@ public void deleteMetadata(ServiceContext context, String metadataId, IDeletionS deleteMetadataFromDB(context, metadataId); getSearchManager().deleteByUuid(findOne.getUuid(), submittor); } else { - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId)); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submittor); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index a3ec98a249d..6bbadffa383 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -653,7 +653,7 @@ public void cancelEditingSession(ServiceContext context, String id) throws Excep // --- remove metadata xmlSerializer.delete(id, ServiceContext.get()); - searchManager.deleteByQuery(String.format("+id:%s", id)); + searchManager.deleteByQuery(String.format("+id:%s", id), DirectDeletionSubmittor.INSTANCE); // Unset METADATA_EDITING_CREATED_DRAFT flag context.getUserSession().removeProperty(Geonet.Session.METADATA_EDITING_CREATED_DRAFT); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 6cbd9512d0d..391620988b4 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -55,6 +55,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; import org.fao.geonet.kernel.search.submission.IIndexSubmittor; @@ -479,6 +480,18 @@ public void handleDeletionResponse(BulkResponse bulkResponse, List docum } } + public void handleDeletionResponse(DeleteByQueryResponse deleteByQueryResponse, String query) { + if (!deleteByQueryResponse.failures().isEmpty()) { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + + throw new RuntimeException(String.format( + "Error during removal of query %s. Errors are '%s'.", query, stringBuilder.toString() + )); + } + } + private void checkIndexResponse(BulkResponse bulkItemResponses, Map documents) throws IOException { if (bulkItemResponses.errors()) { @@ -810,7 +823,7 @@ public Map getFieldsValues(String id, Set fields, String public void clearIndex() throws Exception { - client.deleteByQuery(defaultIndex, "*:*"); + deleteByQuery("*:*", DirectDeletionSubmittor.INSTANCE); } static ImmutableSet docsChangeIncludedFields; @@ -868,13 +881,13 @@ public ISODate getDocChangeDate(String mdId) throws Exception { } @Override - public void deleteByQuery(String txt) throws Exception { - client.deleteByQuery(defaultIndex, txt); + public void deleteByQuery(String query, IDeletionSubmittor submittor) throws Exception { + submittor.submitQueryToIndex(query, this); } @Override public void deleteByUuid(String uuid, IDeletionSubmittor submittor) throws Exception { - submittor.submitToIndex(uuid, this); + submittor.submitUUIDToIndex(uuid, this); } public long getNumDocs(String query) throws Exception { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index 324daf51749..b1fb7b97772 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -73,7 +73,7 @@ boolean rebuildIndex(ServiceContext context, /** * deletes a document by a query. */ - void deleteByQuery(String query) throws Exception; + void deleteByQuery(String query, IDeletionSubmittor submittor) throws Exception; /** * deletes a document by its uuid. diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java index eb1c1546dc5..f99073b575e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java @@ -2,6 +2,8 @@ import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.BulkResponse; +import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; +import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; import org.fao.geonet.index.es.EsRestClient; import org.fao.geonet.kernel.search.EsSearchManager; @@ -15,7 +17,7 @@ public class DirectDeletionSubmittor implements IDeletionSubmittor { private DirectDeletionSubmittor() {} @Override - public void submitToIndex(String uuid, EsSearchManager searchManager) throws IOException { + public void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException { EsRestClient restClient = searchManager.getClient(); List documents = Collections.singletonList(uuid); @@ -24,4 +26,14 @@ public void submitToIndex(String uuid, EsSearchManager searchManager) throws IOE searchManager.handleDeletionResponse(bulkItemResponses, documents); } + + @Override + public void submitQueryToIndex(String query, EsSearchManager searchManager) throws IOException { + EsRestClient restClient = searchManager.getClient(); + + DeleteByQueryRequest deleteByQueryRequest = restClient.buildDeleteByQuery(searchManager.getDefaultIndex(), query); + final DeleteByQueryResponse deleteByQueryResponse = restClient.getClient().deleteByQuery(deleteByQueryRequest); + + searchManager.handleDeletionResponse(deleteByQueryResponse, query); + } } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java index 30384dabff2..a5e744bacc4 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java @@ -6,5 +6,7 @@ public interface IDeletionSubmittor { - void submitToIndex(String uuid, EsSearchManager searchManager) throws IOException; + void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException; + + void submitQueryToIndex(String query, EsSearchManager searchManager) throws IOException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java index 10387ef24f6..ffa97abad20 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java @@ -1,6 +1,8 @@ package org.fao.geonet.kernel.search.submission.batch; import co.elastic.clients.elasticsearch.core.BulkRequest; +import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; +import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; import org.fao.geonet.index.es.EsRestClient; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; @@ -30,6 +32,15 @@ private void deleteDocumentsFromIndex(List toDelete) { .thenAcceptAsync(bulkItemResponses -> searchManager.handleDeletionResponse(bulkItemResponses, toDelete)); queueFuture(currentIndexFuture); } + + private void deleteDocumentByQuery(String query) { + EsRestClient restClient = searchManager.getClient(); + + DeleteByQueryRequest deleteByQueryRequest = restClient.buildDeleteByQuery(searchManager.getDefaultIndex(), query); + final CompletableFuture deleteByQueryFuture = restClient.getAsyncClient().deleteByQuery(deleteByQueryRequest) + .thenAcceptAsync(response -> searchManager.handleDeletionResponse(response, query)); + queueFuture(deleteByQueryFuture); + } } public BatchingDeletionSubmittor() { @@ -41,7 +52,7 @@ public BatchingDeletionSubmittor(int estimatedTotalSize) { } @Override - public void submitToIndex(String uuid, EsSearchManager searchManager) { + public void submitUUIDToIndex(String uuid, EsSearchManager searchManager) { if (state.closed) { throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); } @@ -55,4 +66,14 @@ public void submitToIndex(String uuid, EsSearchManager searchManager) { state.deleteDocumentsFromIndex(toDelete); } } + + @Override + public void submitQueryToIndex(String query, EsSearchManager searchManager) { + if (state.closed) { + throw new IllegalStateException("Attempting to use a closed " + this.getClass().getSimpleName()); + } + + state.searchManager = searchManager; + state.deleteDocumentByQuery(query); + } } diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 919deac8438..12266a5af55 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -250,27 +250,13 @@ public BulkRequest buildDeleteBulkRequest(String index, List deletionUUI return requestBuilder.build(); } - public void deleteByQuery(String index, String query) throws IOException { + public DeleteByQueryRequest buildDeleteByQuery(String index, String query) { checkActivated(); - DeleteByQueryRequest request = DeleteByQueryRequest.of( + return DeleteByQueryRequest.of( b -> b.index(index) .q(query) .refresh(true)); - - final DeleteByQueryResponse deleteByQueryResponse = - client.deleteByQuery(request); - - - if (!deleteByQueryResponse.failures().isEmpty()) { - StringBuilder stringBuilder = new StringBuilder(); - - deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); - - throw new IOException(String.format( - "Error during removal. Errors are '%s'.", stringBuilder.toString() - )); - } } diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java index dcc1b25cbd1..0d50c34dc44 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java @@ -26,10 +26,7 @@ import co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester; import co.elastic.clients.elasticsearch._helpers.bulk.BulkListener; import co.elastic.clients.elasticsearch._types.Result; -import co.elastic.clients.elasticsearch.core.BulkRequest; -import co.elastic.clients.elasticsearch.core.BulkResponse; -import co.elastic.clients.elasticsearch.core.IndexRequest; -import co.elastic.clients.elasticsearch.core.IndexResponse; +import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.util.BinaryData; import co.elastic.clients.util.ContentType; import com.fasterxml.jackson.core.JsonProcessingException; @@ -224,11 +221,11 @@ public void deleteFeatures(String url, String typeName, EsRestClient client) { new Object[]{url, typeName, index, indexType}); try { long begin = System.currentTimeMillis(); - client.deleteByQuery(index, String.format("+featureTypeId:\"%s\"", getIdentifier(url, typeName))); + deleteByQuery(String.format("+featureTypeId:\"%s\"", getIdentifier(url, typeName))); LOGGER.info(" Features deleted in {} ms.", System.currentTimeMillis() - begin); begin = System.currentTimeMillis(); - client.deleteByQuery(index, String.format("+id:\"%s\"", + deleteByQuery(String.format("+id:\"%s\"", getIdentifier(url, typeName))); LOGGER.info(" Report deleted in {} ms.", System.currentTimeMillis() - begin); @@ -238,6 +235,21 @@ public void deleteFeatures(String url, String typeName, EsRestClient client) { } } + private void deleteByQuery(String query) throws IOException { + DeleteByQueryRequest deleteByQueryRequest = client.buildDeleteByQuery(index, query); + DeleteByQueryResponse deleteByQueryResponse = client.getClient().deleteByQuery(deleteByQueryRequest); + if (!deleteByQueryResponse.failures().isEmpty()) { + StringBuilder stringBuilder = new StringBuilder(); + + deleteByQueryResponse.failures().forEach(f -> stringBuilder.append(f.toString())); + + throw new RuntimeException(String.format( + "Error during removal of query %s. Errors are '%s'.", query, stringBuilder.toString() + )); + } + + } + interface TitleResolver { void setTitle(ObjectNode objectNode, SimpleFeature simpleFeature); } From 9cd52424f39487733d84ffe94c648be6c2f921ae Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Fri, 6 Dec 2024 16:29:14 +0100 Subject: [PATCH 15/18] submittor -> submitter --- .../org/fao/geonet/kernel/DataManager.java | 7 ++-- .../fao/geonet/kernel/IndexMetadataTask.java | 4 +-- .../kernel/datamanager/IMetadataIndexer.java | 5 ++- .../kernel/datamanager/IMetadataManager.java | 8 ++--- .../datamanager/base/BaseMetadataIndexer.java | 14 ++++---- .../datamanager/base/BaseMetadataManager.java | 34 +++++++++---------- .../datamanager/base/BaseMetadataStatus.java | 8 ++--- .../datamanager/base/BaseMetadataUtils.java | 8 ++--- .../draft/DraftMetadataManager.java | 8 ++--- .../datamanager/draft/DraftMetadataUtils.java | 8 ++--- .../org/fao/geonet/kernel/mef/Importer.java | 8 ++--- .../geonet/kernel/search/EsSearchManager.java | 24 ++++++------- .../geonet/kernel/search/ISearchManager.java | 12 +++---- .../index/BatchOpsMetadataReindexer.java | 4 +-- .../kernel/search/index/IndexingTask.java | 4 +-- ...ttor.java => DirectDeletionSubmitter.java} | 6 ++-- ...bmittor.java => DirectIndexSubmitter.java} | 8 ++--- ...Submittor.java => IDeletionSubmitter.java} | 2 +- ...dexSubmittor.java => IIndexSubmitter.java} | 2 +- ...or.java => BatchingDeletionSubmitter.java} | 10 +++--- ...ittor.java => BatchingIndexSubmitter.java} | 11 +++--- ...orBase.java => BatchingSubmitterBase.java} | 6 ++-- .../geonet/AbstractCoreIntegrationTest.java | 7 ++-- ...ctIntegrationTestWithMockedSingletons.java | 4 +-- .../kernel/DataManagerIntegrationTest.java | 4 +-- .../kernel/LocalXLinksUpdateDeleteTest.java | 10 +++--- .../geonet/kernel/url/UrlAnalyzerTest.java | 8 ++--- .../fao/geonet/component/csw/Transaction.java | 10 +++--- .../csw/CswTransactionIntegrationTest.java | 7 ++-- .../geonet/kernel/harvest/BaseAligner.java | 8 ++--- .../harvest/harvester/AbstractHarvester.java | 6 ++-- .../harvester/arcsde/ArcSDEHarvester.java | 14 ++++---- .../kernel/harvest/harvester/csw/Aligner.java | 12 +++---- .../harvester/fragment/FragmentHarvester.java | 10 +++--- .../harvest/harvester/geoPREST/Aligner.java | 12 +++---- .../harvest/harvester/geonet/Aligner.java | 12 +++---- .../harvest/harvester/geonet20/Aligner.java | 12 +++---- .../LocalFilesystemHarvester.java | 16 ++++----- .../harvest/harvester/oaipmh/Harvester.java | 8 ++--- .../harvest/harvester/ogcwxs/Harvester.java | 10 +++--- .../harvest/harvester/simpleurl/Aligner.java | 10 +++--- .../harvest/harvester/thredds/Harvester.java | 17 +++++----- .../harvest/harvester/webdav/Harvester.java | 6 ++-- .../harvester/wfsfeatures/Harvester.java | 6 ++-- .../harvester/InspireAtomHarvester.java | 8 ++--- .../listener/metadata/draft/DraftCreated.java | 6 ++-- .../listener/metadata/draft/DraftRemoved.java | 5 ++- .../metadata/draft/DraftUtilities.java | 4 +-- .../listener/metadata/draft/RetireRecord.java | 6 ++-- .../metadata/draft/UpdateOperations.java | 6 ++-- .../api/processing/DatabaseProcessUtils.java | 12 ++----- .../api/processing/XslProcessUtils.java | 4 +-- .../api/records/MetadataInsertDeleteApi.java | 10 +++--- .../geonet/api/records/MetadataSampleApi.java | 5 ++- .../api/records/MetadataSharingApi.java | 8 ++--- .../fao/geonet/api/records/MetadataUtils.java | 4 +-- .../api/records/MetadataValidateApi.java | 5 ++- .../api/records/MetadataWorkflowApi.java | 8 ++--- .../api/records/editing/AjaxEditUtils.java | 6 ++-- .../records/editing/MetadataEditingApi.java | 6 ++-- .../geonet/api/registries/DirectoryUtils.java | 5 ++- .../api/selections/UserSelectionsApi.java | 8 ++--- .../validator/InspireValidationRunnable.java | 8 ++--- .../validator/MInspireEtfValidateProcess.java | 6 ++-- .../api/records/MetadataValidateApiTest.java | 6 ++-- .../records/extent/MetadataExtentApiTest.java | 8 ++--- .../formatters/AbstractFormatterTest.java | 4 +-- .../cache/FormatterCacheIntegrationTest.java | 4 +-- .../api/selections/UserSelectionsApiTest.java | 9 +++-- .../BatchOpsMetadatReindexerTest.java | 9 +++-- 70 files changed, 269 insertions(+), 311 deletions(-) rename core/src/main/java/org/fao/geonet/kernel/search/submission/{DirectDeletionSubmittor.java => DirectDeletionSubmitter.java} (88%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/{DirectIndexSubmittor.java => DirectIndexSubmitter.java} (79%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/{IDeletionSubmittor.java => IDeletionSubmitter.java} (89%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/{IIndexSubmittor.java => IIndexSubmitter.java} (87%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/batch/{BatchingDeletionSubmittor.java => BatchingDeletionSubmitter.java} (87%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/batch/{BatchingIndexSubmittor.java => BatchingIndexSubmitter.java} (88%) rename core/src/main/java/org/fao/geonet/kernel/search/submission/batch/{BatchingSubmittorBase.java => BatchingSubmitterBase.java} (91%) diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index 4564bbd2994..9303b4d5e9c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -55,7 +55,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.UserGroupRepository; import org.jdom.Element; import org.slf4j.Logger; @@ -66,7 +66,6 @@ import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.io.IOException; import java.nio.file.Path; import java.util.Collection; import java.util.List; @@ -160,7 +159,7 @@ public void indexMetadata(final List metadataIds) throws Exception { } @Deprecated - public void indexMetadata(final String metadataId, IIndexSubmittor indexSubmittor) throws Exception { + public void indexMetadata(final String metadataId, IIndexSubmitter indexSubmittor) throws Exception { metadataIndexer.indexMetadata(metadataId, indexSubmittor, IndexingMode.full); } @@ -357,7 +356,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Deprecated public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception { return metadataManager.insertMetadata(context, newMetadata, metadataXml, indexingMode, updateFixedInfo, updateDatestamp, fullRightsForGroup, indexSubmittor); diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index 4b1e2c858bd..bd801fdbb78 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -28,7 +28,7 @@ import org.fao.geonet.Util; import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.User; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.transaction.TransactionStatus; @@ -71,7 +71,7 @@ public IndexMetadataTask(@Nonnull ServiceContext context, @Nonnull List metad } public void run() { - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(_metadataIds.size())) { + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(_metadataIds.size())) { _context.setAsThreadLocal(); while (_transactionStatus != null && !_transactionStatus.isCompleted()) { try { diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java index 190f09aca1f..1c95eac1ec5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java @@ -23,12 +23,11 @@ package org.fao.geonet.kernel.datamanager; -import java.io.IOException; import java.util.List; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.jdom.Element; import org.springframework.data.jpa.domain.Specification; @@ -81,7 +80,7 @@ public interface IMetadataIndexer { */ void indexMetadata(List metadataIds) throws Exception; - void indexMetadata(String metadataId, IIndexSubmittor indexSubmittor, IndexingMode indexingMode) throws Exception; + void indexMetadata(String metadataId, IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception; void indexMetadataPrivileges(String uuid, int id) throws Exception; diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java index dfcdbb201f5..f9bf29b70e5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataManager.java @@ -33,8 +33,8 @@ import org.fao.geonet.kernel.EditLib; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.BatchUpdateQuery; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -73,7 +73,7 @@ public interface IMetadataManager { * @param deletionSubmittor * @throws Exception */ - void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor deletionSubmittor) throws Exception; + void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter deletionSubmittor) throws Exception; /** * Delete the record with the id metadataId @@ -155,7 +155,7 @@ String insertMetadata(ServiceContext context, String schema, Element metadataXml * @throws Exception */ AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, - boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) + boolean updateFixedInfo, UpdateDatestamp updateDatestamp, boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception; /** diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java index 830bef0269c..95225e43b9c 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java @@ -49,9 +49,9 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexFields; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -169,7 +169,7 @@ public int batchDeleteMetadataAndUpdateIndex(Specification Integer.toString(input.getId())).collect(Collectors.toList())); // metadataManager.deleteAll(specification); // So delete one by one even if slower - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(metadataToDelete.size())) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(metadataToDelete.size())) { metadataToDelete.forEach(md -> { try { // Extract information for RecordDeletedEvent @@ -178,7 +178,7 @@ public int batchDeleteMetadataAndUpdateIndex(Specification metadataIds) @Override public void indexMetadata(final List metadataIds) throws Exception { - try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(metadataIds.size())) { + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(metadataIds.size())) { for (String metadataId : metadataIds) { indexMetadata(metadataId, indexSubmittor, IndexingMode.full); } @@ -301,7 +301,7 @@ public void indexMetadata(final List metadataIds) throws Exception { @Override public void indexMetadata(final String metadataId, - final IIndexSubmittor indexSubmittor, + final IIndexSubmitter indexSubmittor, final IndexingMode indexingMode) throws Exception { AbstractMetadata fullMd; diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 055111d8627..0554b9b3bd7 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -50,11 +50,11 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.kernel.setting.SettingInfo; @@ -252,9 +252,9 @@ public void synchronizeDbWithIndex(ServiceContext context) throws Exception { } // remove from index metadata not in DBMS - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String id : docs.keySet()) { - getSearchManager().deleteByQuery(String.format("+id:%s", id), submittor); + getSearchManager().deleteByQuery(String.format("+id:%s", id), submitter); LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); } } @@ -335,16 +335,16 @@ private XmlSerializer getXmlSerializer() { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor submittor) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter submitter) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { boolean isMetadata = findOne.getDataInfo().getType() == MetadataType.METADATA; deleteMetadataFromDB(context, metadataId); // --- update search criteria - getSearchManager().deleteByUuid(findOne.getUuid(), submittor); + getSearchManager().deleteByUuid(findOne.getUuid(), submitter); } else { - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submittor); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submitter); } } @@ -374,7 +374,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit RecordDeletedEvent recordDeletedEvent = new RecordDeletedEvent( metadata.getId(), metadata.getUuid(), new LinkedHashMap<>(), context.getUserSession().getUserIdAsInt(), metadata.getData()); - deleteMetadata(context, metadataId, DirectDeletionSubmittor.INSTANCE); + deleteMetadata(context, metadataId, DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); } @@ -386,7 +386,7 @@ public void purgeMetadata(ServiceContext context, String metadataId, boolean wit @Override public void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), DirectDeletionSubmittor.INSTANCE); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), DirectDeletionSubmitter.INSTANCE); } /** @@ -463,7 +463,7 @@ public boolean apply(@Nullable MetadataCategory input) { newMetadata.getMetadataCategories().addAll(filteredCategories); int finalId = insertMetadata(context, newMetadata, xml, IndexingMode.full, true, UpdateDatestamp.YES, - fullRightsForGroup, DirectIndexSubmittor.INSTANCE).getId(); + fullRightsForGroup, DirectIndexSubmitter.INSTANCE).getId(); return String.valueOf(finalId); } @@ -568,7 +568,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta boolean fullRightsForGroup = false; int finalId = insertMetadata(context, newMetadata, metadataXml, indexingMode, ufo, UpdateDatestamp.NO, - fullRightsForGroup, DirectIndexSubmittor.INSTANCE).getId(); + fullRightsForGroup, DirectIndexSubmitter.INSTANCE).getId(); return String.valueOf(finalId); } @@ -576,7 +576,7 @@ public String insertMetadata(ServiceContext context, String schema, Element meta @Override public AbstractMetadata insertMetadata(ServiceContext context, AbstractMetadata newMetadata, Element metadataXml, IndexingMode indexingMode, boolean updateFixedInfo, UpdateDatestamp updateDatestamp, - boolean fullRightsForGroup, IIndexSubmittor indexSubmittor) throws Exception { + boolean fullRightsForGroup, IIndexSubmitter indexSubmittor) throws Exception { final String schema = newMetadata.getDataInfo().getSchemaId(); // Check if the schema is allowed by settings @@ -786,9 +786,9 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context if (indexingMode != IndexingMode.none) { // Delete old record if UUID changed if (uuidBeforeUfo != null && !uuidBeforeUfo.equals(uuid)) { - getSearchManager().deleteByUuid(uuidBeforeUfo, DirectDeletionSubmittor.INSTANCE); + getSearchManager().deleteByUuid(uuidBeforeUfo, DirectDeletionSubmitter.INSTANCE); } - metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, indexingMode); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE, indexingMode); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java index 1011cc3ffc8..27aaec71bf9 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataStatus.java @@ -38,7 +38,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataStatus; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataStatusRepository; @@ -156,7 +156,7 @@ public String getCurrentStatus(int metadataId) throws Exception { public MetadataStatus setStatus(ServiceContext context, int id, int status, ISODate changeDate, String changeMessage) throws Exception { MetadataStatus statusObject = setStatusExt(context, id, status, changeDate, changeMessage); - metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); return statusObject; } @@ -164,7 +164,7 @@ public MetadataStatus setStatus(ServiceContext context, int id, int status, ISOD public MetadataStatus setStatusExt(MetadataStatus metatatStatus, boolean updateIndex) throws Exception { metadataStatusRepository.save(metatatStatus); if (updateIndex) { - metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(metatatStatus.getMetadataId() + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } return metatatStatus; } @@ -251,7 +251,7 @@ public void changeCurrentStatus(Integer userId, Integer metadataId, Integer newS metatatStatus.setTitles(metadataUtils.extractTitles(Integer.toString(metadataId))); metadataStatusRepository.save(metatatStatus); - metadataIndexer.indexMetadata(metadataId + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } // Utility to verify workflow status transitions diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java index 2e941b07505..05e050662d2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataUtils.java @@ -43,7 +43,7 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.search.index.IndexingList; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.lib.Lib; @@ -513,7 +513,7 @@ public String getNewVersion(String id) { @Override public void setTemplate(final int id, final MetadataType type, final String title) throws Exception { setTemplateExt(id, type); - metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } @Override @@ -527,7 +527,7 @@ public void setTemplateExt(final int id, final MetadataType metadataType) throws @Override public void setHarvested(int id, String harvestUuid) throws Exception { setHarvestedExt(id, harvestUuid); - metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(Integer.toString(id), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } /** @@ -768,7 +768,7 @@ private void transformMd(ServiceContext context, String metadataId, Element md, getXmlSerializer().update(metadataId, md, changeDate, true, uuid, context); if (indexAfterChange) { - metadataIndexer.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java index 54ad4f4f50e..2850a1ed6b9 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataManager.java @@ -32,7 +32,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.repository.PathSpec; import org.fao.geonet.repository.Updater; @@ -70,7 +70,7 @@ public void init(ServiceContext context, Boolean force) throws Exception { * Removes a metadata. */ @Override - public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmittor submittor) throws Exception { + public void deleteMetadata(ServiceContext context, String metadataId, IDeletionSubmitter submitter) throws Exception { AbstractMetadata findOne = metadataUtils.findOne(metadataId); if (findOne != null) { @@ -87,9 +87,9 @@ public void deleteMetadata(ServiceContext context, String metadataId, IDeletionS } deleteMetadataFromDB(context, metadataId); - getSearchManager().deleteByUuid(findOne.getUuid(), submittor); + getSearchManager().deleteByUuid(findOne.getUuid(), submitter); } else { - getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submittor); + getSearchManager().deleteByQuery(String.format("+id:%s", metadataId), submitter); } } diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java index 6bbadffa383..f1b4ca1b0f3 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/draft/DraftMetadataUtils.java @@ -41,8 +41,8 @@ import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; @@ -523,7 +523,7 @@ protected String createDraft(ServiceContext context, String templateId, String g try { newMetadata = (MetadataDraft) metadataManager.insertMetadata(context, newMetadata, xml, IndexingMode.full, true, - UpdateDatestamp.YES, false, DirectIndexSubmittor.INSTANCE); + UpdateDatestamp.YES, false, DirectIndexSubmitter.INSTANCE); Integer finalId = newMetadata.getId(); @@ -653,7 +653,7 @@ public void cancelEditingSession(ServiceContext context, String id) throws Excep // --- remove metadata xmlSerializer.delete(id, ServiceContext.get()); - searchManager.deleteByQuery(String.format("+id:%s", id), DirectDeletionSubmittor.INSTANCE); + searchManager.deleteByQuery(String.format("+id:%s", id), DirectDeletionSubmitter.INSTANCE); // Unset METADATA_EDITING_CREATED_DRAFT flag context.getUserSession().removeProperty(Geonet.Session.METADATA_EDITING_CREATED_DRAFT); diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java index 7ee7b6e008c..7e15f9435d6 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java @@ -45,8 +45,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.datamanager.IMetadataValidator; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -446,7 +446,7 @@ public void apply(@Nonnull final Metadata metadata) { } } - dm.indexMetadata(metadataIdMap.get(index), DirectIndexSubmittor.INSTANCE); + dm.indexMetadata(metadataIdMap.get(index), DirectIndexSubmitter.INSTANCE); } // -------------------------------------------------------------------- @@ -573,7 +573,7 @@ public static void importRecord(String uuid, MEFLib.UuidAction uuidAction, List< if (Log.isDebugEnabled(Geonet.MEF)) { Log.debug(Geonet.MEF, "Deleting existing metadata with UUID : " + uuid); } - metadataManager.deleteMetadata(context, dm.getMetadataId(uuid), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(context, dm.getMetadataId(uuid), DirectDeletionSubmitter.INSTANCE); metadataManager.flush(); } else { throw new UnAuthorizedException("User has no privilege to replace existing metadata", null); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 391620988b4..355cd7f86f2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -55,10 +55,10 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.index.OverviewIndexFieldUpdater; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -429,7 +429,7 @@ public void updateFieldAsynch(String id, String field, Object value) { public void index(Path schemaDir, Element metadata, String id, Multimap dbFields, MetadataType metadataType, - IIndexSubmittor indexSubmittor, + IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception { Element docs = new Element("doc"); @@ -761,7 +761,7 @@ public boolean rebuildIndex(ServiceContext context, } } } - try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(listOfIdsToIndex.size())) { + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(listOfIdsToIndex.size())) { for (String id : listOfIdsToIndex) { metadataIndexer.indexMetadata(id, indexSubmittor, IndexingMode.full); } @@ -773,7 +773,7 @@ public boolean rebuildIndex(ServiceContext context, final List metadataIds = metadataRepository.findAllIdsBy( Specification.where(metadataSpec) ); - try (BatchingIndexSubmittor indexSubmittor = new BatchingIndexSubmittor(metadataIds.size())) { + try (BatchingIndexSubmitter indexSubmittor = new BatchingIndexSubmitter(metadataIds.size())) { for (Integer id : metadataIds) { metadataIndexer.indexMetadata(id + "", indexSubmittor, IndexingMode.full); } @@ -823,7 +823,7 @@ public Map getFieldsValues(String id, Set fields, String public void clearIndex() throws Exception { - deleteByQuery("*:*", DirectDeletionSubmittor.INSTANCE); + deleteByQuery("*:*", DirectDeletionSubmitter.INSTANCE); } static ImmutableSet docsChangeIncludedFields; @@ -881,13 +881,13 @@ public ISODate getDocChangeDate(String mdId) throws Exception { } @Override - public void deleteByQuery(String query, IDeletionSubmittor submittor) throws Exception { - submittor.submitQueryToIndex(query, this); + public void deleteByQuery(String query, IDeletionSubmitter submitter) throws Exception { + submitter.submitQueryToIndex(query, this); } @Override - public void deleteByUuid(String uuid, IDeletionSubmittor submittor) throws Exception { - submittor.submitUUIDToIndex(uuid, this); + public void deleteByUuid(String uuid, IDeletionSubmitter submitter) throws Exception { + submitter.submitUUIDToIndex(uuid, this); } public long getNumDocs(String query) throws Exception { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java index b1fb7b97772..53ccd3caed1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -28,8 +28,8 @@ import jeeves.server.context.ServiceContext; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.MetadataType; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.jdom.Element; import java.io.IOException; @@ -49,10 +49,10 @@ public interface ISearchManager { /** * Indexes a metadata record. * - * @param indexSubmittor The submittor to use + * @param indexSubmittor The submitter to use */ void index(Path schemaDir, Element metadata, String id, Multimap moreFields, - MetadataType metadataType, IIndexSubmittor indexSubmittor, IndexingMode indexingMode) + MetadataType metadataType, IIndexSubmitter indexSubmittor, IndexingMode indexingMode) throws Exception; @@ -73,12 +73,12 @@ boolean rebuildIndex(ServiceContext context, /** * deletes a document by a query. */ - void deleteByQuery(String query, IDeletionSubmittor submittor) throws Exception; + void deleteByQuery(String query, IDeletionSubmitter submitter) throws Exception; /** * deletes a document by its uuid. */ - void deleteByUuid(String uuid, IDeletionSubmittor submittor) throws Exception; + void deleteByUuid(String uuid, IDeletionSubmitter submitter) throws Exception; boolean isIndexWritable(String indexName) throws IOException, ElasticsearchException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java index 623798dc650..be36b13e66e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/BatchOpsMetadataReindexer.java @@ -49,7 +49,7 @@ import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.MetadataIndexerProcessor; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.util.ThreadUtils; import org.fao.geonet.utils.Log; import org.springframework.jmx.export.MBeanExporter; @@ -227,7 +227,7 @@ public void run() { "Indexing range [%d-%d]/%d by threads %s.", beginIndex, beginIndex + count, ids.length, Thread.currentThread().getId())); long start = System.currentTimeMillis(); - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(count)) { + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(count)) { for (int i = beginIndex; i < beginIndex + count; i++) { try { dm.indexMetadata(ids[i] + "", batchingIndexSubmittor); diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index 40f747b491c..8fd22e30ae1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -29,7 +29,7 @@ import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.utils.Log; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; @@ -71,7 +71,7 @@ private void indexRecords() { Log.debug(Geonet.INDEX_ENGINE, "Indexing task / List of records to index: " + metadataIdentifiers.toString() + "."); } - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(metadataIdentifiers.size())) { + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(metadataIdentifiers.size())) { for (Integer metadataIdentifier : metadataIdentifiers) { try { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java similarity index 88% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java index f99073b575e..c8dc445ede4 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectDeletionSubmitter.java @@ -11,10 +11,10 @@ import java.util.Collections; import java.util.List; -public class DirectDeletionSubmittor implements IDeletionSubmittor { - public static final DirectDeletionSubmittor INSTANCE = new DirectDeletionSubmittor(); +public class DirectDeletionSubmitter implements IDeletionSubmitter { + public static final DirectDeletionSubmitter INSTANCE = new DirectDeletionSubmitter(); - private DirectDeletionSubmittor() {} + private DirectDeletionSubmitter() {} @Override public void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java similarity index 79% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java index 2367cf812bd..9ac32a00668 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/DirectIndexSubmitter.java @@ -10,12 +10,12 @@ import java.util.Map; /** - * An index submittor that directly and synchronously transmits new documents to the index + * An index submitter that directly and synchronously transmits new documents to the index */ -public class DirectIndexSubmittor implements IIndexSubmittor { - public static final DirectIndexSubmittor INSTANCE = new DirectIndexSubmittor(); +public class DirectIndexSubmitter implements IIndexSubmitter { + public static final DirectIndexSubmitter INSTANCE = new DirectIndexSubmitter(); - private DirectIndexSubmittor() {} + private DirectIndexSubmitter() {} @Override public void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java similarity index 89% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java index a5e744bacc4..745a8bb706e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IDeletionSubmitter.java @@ -4,7 +4,7 @@ import java.io.IOException; -public interface IDeletionSubmittor { +public interface IDeletionSubmitter { void submitUUIDToIndex(String uuid, EsSearchManager searchManager) throws IOException; diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java similarity index 87% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java index d65d65a7350..11fd9e26f8e 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/IIndexSubmitter.java @@ -4,7 +4,7 @@ import java.io.IOException; -public interface IIndexSubmittor { +public interface IIndexSubmitter { void submitToIndex(String id, String jsonDocument, EsSearchManager searchManager) throws IOException; } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java similarity index 87% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java index ffa97abad20..3fe31ffda76 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingDeletionSubmitter.java @@ -2,17 +2,15 @@ import co.elastic.clients.elasticsearch.core.BulkRequest; import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; -import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; import org.fao.geonet.index.es.EsRestClient; import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.IDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.IDeletionSubmitter; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -public class BatchingDeletionSubmittor extends BatchingSubmittorBase implements IDeletionSubmittor { +public class BatchingDeletionSubmitter extends BatchingSubmitterBase implements IDeletionSubmitter { protected static class State extends StateBase { private final List listOfUUIDsToDelete = new ArrayList<>(); @@ -43,11 +41,11 @@ private void deleteDocumentByQuery(String query) { } } - public BatchingDeletionSubmittor() { + public BatchingDeletionSubmitter() { super(new State()); } - public BatchingDeletionSubmittor(int estimatedTotalSize) { + public BatchingDeletionSubmitter(int estimatedTotalSize) { super(new State(), estimatedTotalSize); } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java similarity index 88% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java index 45537ca97d4..80a9f18f0c4 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmittor.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingIndexSubmitter.java @@ -3,21 +3,20 @@ import co.elastic.clients.elasticsearch.core.BulkRequest; import org.fao.geonet.index.es.EsRestClient; import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import java.io.IOException; import java.io.UncheckedIOException; -import java.lang.ref.Cleaner; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; /** * An index submitter that batches documents into larger chunks and sends them asynchronously to the index. - * When closing, this submittor sends the remaining item to the index and waits until all elasticsearch requests have been received, + * When closing, this submitter sends the remaining item to the index and waits until all elasticsearch requests have been received, * so after closing there are no pending changes */ -public class BatchingIndexSubmittor extends BatchingSubmittorBase implements IIndexSubmittor { +public class BatchingIndexSubmitter extends BatchingSubmitterBase implements IIndexSubmitter { protected static class State extends StateBase { private final Map listOfDocumentsToIndex = new HashMap<>(); @@ -49,14 +48,14 @@ private void sendDocumentsToIndex(Map toIndex) { } } - public BatchingIndexSubmittor() { + public BatchingIndexSubmitter() { super(new State()); } /** * @param estimatedTotalSize The estimated size of documents to index. Does not need to match the actual amount of submitted documents */ - public BatchingIndexSubmittor(int estimatedTotalSize) { + public BatchingIndexSubmitter(int estimatedTotalSize) { super(new State(), estimatedTotalSize); } diff --git a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java similarity index 91% rename from core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java rename to core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java index cabce9bfb73..596896de2a4 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmittorBase.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/submission/batch/BatchingSubmitterBase.java @@ -3,19 +3,19 @@ import java.lang.ref.Cleaner; import java.util.concurrent.CompletableFuture; -public abstract class BatchingSubmittorBase implements AutoCloseable { +public abstract class BatchingSubmitterBase implements AutoCloseable { private static final Cleaner CLEANER = Cleaner.create(); protected final STATE state; protected final int commitInterval; private final Cleaner.Cleanable cleanable; - protected BatchingSubmittorBase(final STATE state) { + protected BatchingSubmitterBase(final STATE state) { this.commitInterval = 200; this.state = state; this.cleanable = CLEANER.register(this, state); } - protected BatchingSubmittorBase(final STATE state, int estimatedTotalSize) { + protected BatchingSubmitterBase(final STATE state, int estimatedTotalSize) { if (estimatedTotalSize < 0) { throw new IllegalArgumentException("estimatedTotalSize must not be negative"); } diff --git a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java index 09627d01e43..3a4178df315 100644 --- a/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/AbstractCoreIntegrationTest.java @@ -49,8 +49,7 @@ import org.fao.geonet.kernel.mef.Importer; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.AbstractSpringDataTest; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; @@ -352,7 +351,7 @@ public int importMetadataXML(ServiceContext context, String uuid, InputStream xm id, createDate, createDate, "" + groupId, metadataType); - dataManager.indexMetadata(id.get(0), DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(id.get(0), DirectIndexSubmitter.INSTANCE); return Integer.parseInt(id.get(0)); } @@ -417,6 +416,6 @@ protected AbstractMetadata injectMetadataInDb(Element sampleMetadataXml, Service .setHarvested(false); return metadataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.none, false, UpdateDatestamp.NO, - false, DirectIndexSubmittor.INSTANCE); + false, DirectIndexSubmitter.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java index e4bb80e5f75..10be099be52 100644 --- a/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java +++ b/core/src/test/java/org/fao/geonet/kernel/AbstractIntegrationTestWithMockedSingletons.java @@ -7,7 +7,7 @@ import org.fao.geonet.domain.MetadataType; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.SourceRepository; import org.jdom.Element; import org.springframework.beans.factory.annotation.Autowired; @@ -66,6 +66,6 @@ protected AbstractMetadata insertTemplateResourceInDb(ServiceContext serviceCont false, NO, false, - DirectIndexSubmittor.INSTANCE); + DirectIndexSubmitter.INSTANCE); } } diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index 745fa29e18e..52120b59dac 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -41,7 +41,7 @@ import org.fao.geonet.domain.User; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -79,7 +79,7 @@ public void testDeleteMetadata() throws Exception { assertEquals(count + 1, metadataRepository.count()); - metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(serviceContext, String.valueOf(mdId), DirectDeletionSubmitter.INSTANCE); assertEquals(count, metadataRepository.count()); } diff --git a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java index 048b3d5cf3c..66dabc7612a 100644 --- a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java @@ -5,7 +5,7 @@ import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Xml; @@ -75,8 +75,8 @@ public void deleteAllowedWhenRefNotExists() throws Exception { AbstractMetadata contactMetadata = insertContact(); AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId()), DirectDeletionSubmittor.INSTANCE); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(serviceContext, Integer.toString(vicinityMapMetadata.getId()), DirectDeletionSubmitter.INSTANCE); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } @@ -88,7 +88,7 @@ public void deleteHasToBeForbiddenWhenRefExistsAndSettingsSaySo() throws Excepti try { metadataManager.deleteMetadata(serviceContext, - Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); + Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); } catch (Exception e) { } @@ -101,7 +101,7 @@ public void deleteHasToBeAllowedWhenRefExistsAndSettingsSaySo() throws Exception AbstractMetadata contactMetadata = insertContact(); insertVicinityMap(contactMetadata); - metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(serviceContext, Integer.toString(contactMetadata.getId()), DirectDeletionSubmitter.INSTANCE); assertNull(metadataManager.getMetadata(Integer.toString(contactMetadata.getId()))); } diff --git a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java index 8f6926eec72..0a5a3c8c1e1 100644 --- a/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/url/UrlAnalyzerTest.java @@ -11,8 +11,8 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.LinkRepository; import org.fao.geonet.repository.LinkStatusRepository; @@ -189,7 +189,7 @@ public void deleteMetadaLinkWhenMdDisappeared() throws Exception { SimpleJpaRepository metadataLinkRepository = new SimpleJpaRepository(MetadataLink.class, entityManager); List metadataLinkList = metadataLinkRepository.findAll(); assertEquals(6, metadataLinkList.size()); - dataManager.deleteMetadata(context, md.getId() + "", DirectDeletionSubmittor.INSTANCE); + dataManager.deleteMetadata(context, md.getId() + "", DirectDeletionSubmitter.INSTANCE); linkRepository.findAll().stream().forEach(toTest::purgeMetataLink); @@ -303,7 +303,7 @@ private AbstractMetadata insertMetadataInDb(Element element) throws Exception { false, NO, false, - DirectIndexSubmittor.INSTANCE); + DirectIndexSubmitter.INSTANCE); return dbInsertedMetadata; } diff --git a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java index 4452031207f..a395ffe1d4c 100644 --- a/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java +++ b/csw-server/src/main/java/org/fao/geonet/component/csw/Transaction.java @@ -50,8 +50,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.utils.Log; @@ -281,7 +281,7 @@ private boolean insertTransaction(Element xml, List documents, dataMan.setOperation(context, id, "" + ReservedGroup.all.getId(), ReservedOperation.dynamic); } - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); AbstractMetadata metadata = metadataUtils.findOne(id); ApplicationContext applicationContext = ApplicationContextHolder.get(); @@ -527,7 +527,7 @@ private int deleteTransaction(Element request, ServiceContext context) throws Ex return deleted; // delete all matching records - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(results.size())) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(results.size())) { while (i.hasNext()) { Element result = i.next(); String uuid = result.getChildText("identifier", Csw.NAMESPACE_DC); @@ -556,7 +556,7 @@ private int deleteTransaction(Element request, ServiceContext context) throws Ex context.getUserSession().getUserIdAsInt(), metadata.getData()); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); recordDeletedEvent.publish(ApplicationContextHolder.get()); deleted++; diff --git a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java index e51a2219344..be50321e7f3 100644 --- a/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java +++ b/csw-server/src/test/java/org/fao/geonet/component/csw/CswTransactionIntegrationTest.java @@ -36,7 +36,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepositoryTest; import org.fao.geonet.utils.Xml; import org.jdom.Content; @@ -49,9 +49,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.fao.geonet.constants.Geonet.Namespaces.GCO; @@ -391,7 +388,7 @@ private void addPhotographicMetadataToRepository(int ownerId) throws Exception { extras.put("_isTemplate", "n"); extras.put("_owner", "" + ownerId); _searchManager.index(schemaDir, metadata.getXmlData(false), "" + metadata.getId(), extras, - MetadataType.METADATA, DirectIndexSubmittor.INSTANCE, IndexingMode.full); + MetadataType.METADATA, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } private Element createUpdateTransaction(String property, Object newValue) { diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java index 99c918fbb9f..f7e3763f81a 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/BaseAligner.java @@ -35,7 +35,7 @@ import org.fao.geonet.kernel.harvest.harvester.CategoryMapper; import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.Privileges; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataCategoryRepository; @@ -69,11 +69,11 @@ public abstract class BaseAligner

extends AbstractAlig private static final Logger LOGGER = LoggerFactory.getLogger(Geonet.HARVESTER); public final AtomicBoolean cancelMonitor; - protected final BatchingIndexSubmittor batchingIndexSubmittor; + protected final BatchingIndexSubmitter batchingIndexSubmitter; protected BaseAligner(AtomicBoolean cancelMonitor) { this.cancelMonitor = cancelMonitor; - this.batchingIndexSubmittor = new BatchingIndexSubmittor(); + this.batchingIndexSubmitter = new BatchingIndexSubmitter(); } public void addCategories(AbstractMetadata metadata, Iterable categories, @@ -204,6 +204,6 @@ public Element translateMetadataContent(ServiceContext context, @Override public void close() throws Exception { - this.batchingIndexSubmittor.close(); + this.batchingIndexSubmitter.close(); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java index d70e366e0f6..e56b9342930 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java @@ -49,7 +49,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.harvest.Common.OperResult; import org.fao.geonet.kernel.harvest.Common.Status; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.HarvesterSettingsManager; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -280,10 +280,10 @@ public void destroy() throws Exception { final Specification ownedByHarvester = Specification.where(MetadataSpecs.hasHarvesterUuid(getParams().getUuid())); Set sources = new HashSet<>(); List allIdsBy = metadataRepository.findAllIdsBy(ownedByHarvester); - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(allIdsBy.size())) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(allIdsBy.size())) { for (Integer metadataId : allIdsBy) { sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId()); - metadataManager.deleteMetadata(context, "" + metadataId, submittor); + metadataManager.deleteMetadata(context, "" + metadataId, submitter); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java index 00e043e1a6a..8d8b31265ea 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java @@ -41,8 +41,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.MetadataSpecs; @@ -307,7 +307,7 @@ private void align(Map metadataList) throws Exception { // Set idsResultHs = Sets.newHashSet(idsForHarvestingResult); List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (Integer existingId : existingMetadata) { if (cancelMonitor.get()) { @@ -315,7 +315,7 @@ private void align(Map metadataList) throws Exception { } if (!idsResultHs.contains(existingId)) { log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString(), submittor); + metadataManager.deleteMetadata(context, existingId.toString(), submitter); result.locallyRemoved++; } } @@ -354,7 +354,7 @@ private void updateMetadata(Element xml, String id, GroupMapper localGroups, fin metadataManager.flush(); - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } /** @@ -398,13 +398,13 @@ private String addMetadata(Element xml, String uuid, String schema, GroupMapper aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); aligner.addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 3bdeb588bff..e0a9359e378 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -57,7 +57,7 @@ import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Xml; import org.jdom.Element; @@ -257,12 +257,12 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { if (!records.contains(uuid)) { String id = localUuids.getID(uuid); log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); result.locallyRemoved++; } } @@ -367,13 +367,13 @@ private void addMetadata(RecordInfo ri, String uuidToAssign) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.addedMetadata++; } @@ -437,7 +437,7 @@ private void updateMetadata(RecordInfo ri, String id, boolean force) throws Exce } else { log.debug(" - Updating local metadata for uuid:" + ri.uuid); if (updatingLocalMetadata(ri, id, force)) { - metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java index 73e1d653d4a..7623fa3233e 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java @@ -403,14 +403,14 @@ private void createSubtemplate(String schema, Element md, String uuid, String ti addCategories(metadata, params.categories, localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); // Note: we use fragmentAllPrivs here because subtemplates need to be // visible/accessible to all addPrivileges(id, fragmentAllPrivs, localGroups, context); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); metadataManager.flush(); @@ -594,7 +594,7 @@ private void update(String id, Element template, String title, boolean isSubtemp } dataMan.setHarvestedExt(iId, params.uuid, Optional.of(harvestUri)); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); metadataManager.flush(); } @@ -634,7 +634,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } metadata.getCategories().add(metadataCategory); } - metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, template, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); @@ -643,7 +643,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } addPrivileges(id, params.privileges, localGroups, context); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); if (log.isDebugEnabled()) { log.debug(" - Commit " + id); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index df979b0c5b0..9e0ed90f72c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -51,7 +51,7 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.utils.GeonetHttpRequestFactory; import org.fao.geonet.utils.Xml; @@ -106,7 +106,7 @@ public HarvestResult align(Set records, List errors) t //----------------------------------------------------------------------- //--- remove old metadata - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { if (cancelMonitor.get()) { return result; @@ -117,7 +117,7 @@ public HarvestResult align(Set records, List errors) t if (log.isDebugEnabled()) log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); metadataManager.flush(); @@ -192,13 +192,13 @@ private void addMetadata(RecordInfo ri) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.addedMetadata++; } @@ -242,7 +242,7 @@ private void updateMetadata(RecordInfo ri, String id) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, null, true); metadataManager.flush(); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java index 292af9b68ee..1b31e15ac82 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java @@ -44,7 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.*; import org.fao.geonet.kernel.mef.*; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.GroupRepository; @@ -151,7 +151,7 @@ public HarvestResult align(SortedSet records, List err //----------------------------------------------------------------------- //--- remove old metadata - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { if (cancelMonitor.get()) { return this.result; @@ -162,7 +162,7 @@ public HarvestResult align(SortedSet records, List err String id = localUuids.getID(uuid); if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id:" + id); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); result.locallyRemoved++; } @@ -518,7 +518,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, this.batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, ufo, UpdateDatestamp.NO, false, this.batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); @@ -547,7 +547,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca } context.getBean(IMetadataManager.class).save(metadata); - dataMan.indexMetadata(id, this.batchingIndexSubmittor); + dataMan.indexMetadata(id, this.batchingIndexSubmitter); result.addedMetadata++; return id; @@ -844,7 +844,7 @@ private void updateMetadata(RecordInfo ri, String id, Element md, metadataManager.save(metadata); // dataMan.flush(); - dataMan.indexMetadata(id, this.batchingIndexSubmittor); + dataMan.indexMetadata(id, this.batchingIndexSubmitter); } private void handleFile(String id, String file, MetadataResourceVisibility visibility, String changeDate, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java index d8ced85f476..0c64cd2844c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java @@ -42,8 +42,8 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.repository.specification.MetadataCategorySpecs; import org.fao.geonet.utils.XmlRequest; @@ -106,7 +106,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { //----------------------------------------------------------------------- //--- remove old metadata - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { if (cancelMonitor.get()) { return this.result; @@ -116,7 +116,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { String id = localUuids.getID(uuid); if (log.isDebugEnabled()) log.debug(" - Removing old metadata with id=" + id); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); metadataManager.flush(); this.result.locallyRemoved++; @@ -161,7 +161,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { //--- maybe the metadata was unretrievable if (id != null) { - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } } @@ -216,7 +216,7 @@ private String addMetadata(Element info) throws Exception { List categories = info.getChildren("category"); addCategories(metadata, categories); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java index 296f61ade62..febd1658f67 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java @@ -44,8 +44,8 @@ import org.fao.geonet.kernel.harvest.harvester.GroupMapper; import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.specification.MetadataSpecs; import org.fao.geonet.utils.IO; @@ -55,8 +55,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import java.util.*; - /** * Harvester for local filesystem. * @@ -114,7 +112,7 @@ private HarvestResult align(Path root) throws Exception { "from the same source if they " + " were not in this harvesting result..."); List existingMetadata = context.getBean(MetadataRepository.class).findIdsBy((Specification) MetadataSpecs.hasHarvesterUuid(params.getUuid())); - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (Integer existingId : existingMetadata) { if (cancelMonitor.get()) { @@ -122,7 +120,7 @@ private HarvestResult align(Path root) throws Exception { } if (!idsResultHs.contains(existingId)) { log.debug(" Removing: " + existingId); - metadataManager.deleteMetadata(context, existingId.toString(), submittor); + metadataManager.deleteMetadata(context, existingId.toString(), submitter); result.locallyRemoved++; } } @@ -176,7 +174,7 @@ void updateMetadata(Element xml, final String id, GroupMapper localGroups, final metadataManager.flush(); - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } String addMetadata(Element xml, String uuid, String schema, GroupMapper localGroups, final CategoryMapper localCateg, @@ -224,7 +222,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro aligner.addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); @@ -233,7 +231,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro metadataManager.flush(); if (index) { - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java index d132c111137..57209b1181f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java @@ -430,7 +430,7 @@ private void addMetadata(XmlRequest t, RecordInfo ri, String processName, Map addMetadata(Element capa) throws Exception { if (!dataMan.existsMetadataUuid(uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(uuid); @@ -837,7 +837,7 @@ private WxSLayerRegistry addLayerMetadata(Element layer, Element capa) throws JD } if (!dataMan.existsMetadataUuid(reg.uuid)) { result.addedMetadata++; - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); } else { result.updatedMetadata++; String id = dataMan.getMetadataId(reg.uuid); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java index 8464f1accbb..141fdb7c78d 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/simpleurl/Aligner.java @@ -44,7 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.HarvestResult; import org.fao.geonet.kernel.harvest.harvester.UUIDMapper; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.jdom.Element; @@ -179,13 +179,13 @@ public HarvestResult cleanupRemovedRecords(Set records) throws Exception return result; } - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { if (!records.contains(uuid)) { String id = localUuids.getID(uuid); log.debug(" - Removing old metadata with local id:" + id); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); result.locallyRemoved++; } } @@ -252,13 +252,13 @@ private void addMetadata(Map.Entry recordInfo, String overrideU addCategories(metadata, params.getCategories(), localCateg, context, null, false); - metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, xml, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - metadataIndexer.indexMetadata(id, batchingIndexSubmittor, IndexingMode.full); + metadataIndexer.indexMetadata(id, batchingIndexSubmitter, IndexingMode.full); result.addedMetadata++; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java index 937dded6297..95f41169f5f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/thredds/Harvester.java @@ -56,8 +56,8 @@ import org.fao.geonet.kernel.harvest.harvester.RecordInfo; import org.fao.geonet.kernel.harvest.harvester.UriMapper; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataCategoryRepository; import org.fao.geonet.util.Sha1Encoder; @@ -92,7 +92,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; -import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; @@ -290,7 +289,7 @@ public HarvestResult harvest(Logger log) throws Exception { harvestCatalog(xml); //--- Remove previously harvested metadata for uris that no longer exist on the remote site - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String localUri : localUris.getUris()) { if (cancelMonitor.get()) { @@ -305,7 +304,7 @@ public HarvestResult harvest(Logger log) throws Exception { if (log.isDebugEnabled()) log.debug(" - Removing deleted metadata with id: " + record.id); - mdManager.deleteMetadata(context, record.id, submittor); + mdManager.deleteMetadata(context, record.id, submitter); if (record.isTemplate.equals("s")) { //--- Uncache xlinks if a subtemplate @@ -577,13 +576,13 @@ private void saveMetadata(Element md, String uuid, String uri, boolean isService addCategories(metadata, params.getCategories(), localCateg, context, null, false); } - metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); + metadata = (Metadata) mdManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); - mdIndexer.indexMetadata(id, DirectIndexSubmittor.INSTANCE, IndexingMode.full); + mdIndexer.indexMetadata(id, DirectIndexSubmitter.INSTANCE, IndexingMode.full); mdManager.flush(); } @@ -789,9 +788,9 @@ private void deleteExistingMetadata(String uri) throws Exception { if (localRecords == null) return; - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor(localRecords.size())) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter(localRecords.size())) { for (RecordInfo record : localRecords) { - mdManager.deleteMetadata(context, record.id, submittor); + mdManager.deleteMetadata(context, record.id, submitter); if (record.isTemplate.equals("s")) { //--- Uncache xlinks if a subtemplate diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java index 0c156465682..e0c7040aa43 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/webdav/Harvester.java @@ -340,14 +340,14 @@ private void addMetadata(RemoteFile rf) throws Exception { } catch (NumberFormatException e) { } - metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmittor); + metadata = metadataManager.insertMetadata(context, metadata, md, IndexingMode.none, false, UpdateDatestamp.NO, false, batchingIndexSubmitter); String id = String.valueOf(metadata.getId()); addPrivileges(id, params.getPrivileges(), localGroups, context); metadataManager.flush(); - dataMan.indexMetadata(id, batchingIndexSubmittor); + dataMan.indexMetadata(id, batchingIndexSubmitter); result.addedMetadata++; } @@ -522,7 +522,7 @@ private void updateMetadata(RemoteFile rf, RecordInfo recordInfo, boolean force) dataMan.flush(); - dataMan.indexMetadata(recordInfo.id, batchingIndexSubmittor); + dataMan.indexMetadata(recordInfo.id, batchingIndexSubmitter); } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java index 4ab3ce8d8b1..b7ac6c320c9 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/wfsfeatures/Harvester.java @@ -44,7 +44,7 @@ import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.FragmentParams; import org.fao.geonet.kernel.harvest.harvester.fragment.FragmentHarvester.HarvestSummary; -import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingDeletionSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.lib.Lib; import org.fao.geonet.utils.*; @@ -335,7 +335,7 @@ public void deleteOrphanedMetadata(Set updatedMetadata) throws Exception if (log.isDebugEnabled()) log.debug(" - Removing orphaned metadata records and fragments after update"); - try (BatchingDeletionSubmittor submittor = new BatchingDeletionSubmittor()) { + try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String uuid : localUuids.getUUIDs()) { try { String isTemplate = localUuids.getTemplate(uuid); @@ -345,7 +345,7 @@ public void deleteOrphanedMetadata(Set updatedMetadata) throws Exception if (!updatedMetadata.contains(uuid)) { String id = localUuids.getID(uuid); - metadataManager.deleteMetadata(context, id, submittor); + metadataManager.deleteMetadata(context, id, submitter); if (isTemplate.equals("s")) { result.subtemplatesRemoved++; diff --git a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java index 8231155be4f..48fbad12059 100644 --- a/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java +++ b/inspire-atom/src/main/java/org/fao/geonet/inspireatom/harvester/InspireAtomHarvester.java @@ -37,7 +37,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.InspireAtomFeedRepository; @@ -170,7 +170,7 @@ public final void harvestServiceMetadata(final ServiceContext context, final Str // Removes all atom information from existing metadata. Harvester will reload with updated information localLogger.info("ATOM feed harvest: remove existing metadata feed"); repository.deleteAll(InspireAtomFeedSpecs.hasMetadataId(Integer.parseInt(metadataId))); - dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); // Process service metadata feeds // datasetsInformation stores the dataset information for identifier and namespace for the services feed. @@ -245,7 +245,7 @@ private List processServiceMetadataFeeds(final DataManager data repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); // Extract datasets information (identifier, namespace) from the service feed: @@ -338,7 +338,7 @@ private void processDatasetsMetadataFeeds(final DataManager dataMan, repository.save(inspireAtomFeed); // Index the metadata to store the atom feed information in the index - dataMan.indexMetadata(metadataId, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(metadataId, DirectIndexSubmitter.INSTANCE); result.addContent(new Element("feed").setAttribute("uuid", metadataUuid) .setAttribute("feed", atomUrl).setAttribute("status", "ok")); } catch (Exception ex) { diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java index b71164128c1..36a1c74e95c 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftCreated.java @@ -23,8 +23,6 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; - import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.MetadataDraft; @@ -32,7 +30,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -69,7 +67,7 @@ public void doAfterCommit(MetadataDraftAdd event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java index 3a2d5839650..ccff7658346 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftRemoved.java @@ -23,7 +23,6 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; import java.util.List; import javax.transaction.Transactional; @@ -36,7 +35,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -69,7 +68,7 @@ public void doAfterCommit(MetadataDraftRemove event) { if (!(md instanceof MetadataDraft)) { Log.trace(Geonet.DATA_MANAGER, "Reindexing " + md.getId()); try { - metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(md.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, e, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java index 68bc35e6bde..d14733fad32 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/DraftUtilities.java @@ -10,7 +10,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.*; import org.fao.geonet.repository.specification.MetadataFileUploadSpecs; import org.fao.geonet.repository.specification.MetadataValidationSpecs; @@ -187,7 +187,7 @@ public void removeDraft(MetadataDraft draft) { // --- remove metadata xmlSerializer.delete(String.valueOf(id), ServiceContext.get()); - searchManager.deleteByUuid(draft.getUuid(), DirectDeletionSubmittor.INSTANCE); + searchManager.deleteByUuid(draft.getUuid(), DirectDeletionSubmitter.INSTANCE); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Couldn't cleanup draft " + draft, e); } diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java index 188df98b695..2aa26f293a1 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/RetireRecord.java @@ -34,7 +34,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.utils.Log; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; @@ -42,8 +42,6 @@ import org.springframework.transaction.event.TransactionPhase; import org.springframework.transaction.event.TransactionalEventListener; -import java.util.Arrays; - /** * When a record gets a status change to retired, unpublish it. * @@ -90,7 +88,7 @@ public void doBeforeCommit(MetadataStatusChanged event) { metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.view.getId()); metadataOperations.forceUnsetOperation(context, event.getMd().getId(), ReservedGroup.all.getId(), ReservedOperation.notify.getId()); - metadataIndexer.indexMetadata(String.valueOf(event.getMd().getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(event.getMd().getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error upgrading status", e); diff --git a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java index 6346a483a28..57e5a93260d 100644 --- a/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java +++ b/listeners/src/main/java/org/fao/geonet/listener/metadata/draft/UpdateOperations.java @@ -23,8 +23,6 @@ package org.fao.geonet.listener.metadata.draft; -import java.util.Arrays; - import org.fao.geonet.constants.Geonet; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.Group; @@ -37,7 +35,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataOperations; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.GroupRepository; import org.fao.geonet.repository.MetadataDraftRepository; import org.fao.geonet.utils.Log; @@ -123,7 +121,7 @@ public void doAfterCommit(MetadataShare event) { metadataOperations.forceSetOperation(context, draft.getId(), op.getId().getGroupId(), op.getId().getOperationId()); } - metadataIndexer.indexMetadata(String.valueOf(draft.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(draft.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e) { Log.error(Geonet.DATA_MANAGER, "Error cascading operation to draft", e); } diff --git a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java index 5bf5092f77e..b32279eee8b 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/DatabaseProcessUtils.java @@ -26,26 +26,18 @@ import jeeves.server.context.ServiceContext; import org.apache.commons.lang.StringUtils; import org.fao.geonet.api.processing.report.MetadataReplacementProcessingReport; -import org.fao.geonet.api.processing.report.XsltMetadataProcessingReport; import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.kernel.AccessManager; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.SchemaManager; -import org.fao.geonet.kernel.datamanager.IMetadataManager; -import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.setting.SettingManager; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataRepository; -import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.utils.Xml; import org.jdom.Element; -import java.util.Map; - public class DatabaseProcessUtils { /** * Process a metadata with a database SQL query. @@ -112,7 +104,7 @@ public static Element process(ServiceContext context, String id, dataMan.updateMetadata(context, id, wellFormedXml, validate, ufo, language, new ISODate().toString(), updateDateStamp, index ? IndexingMode.full : IndexingMode.none); if (index) { - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } } diff --git a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java index cc842613b73..ef1ee211dc8 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java @@ -35,7 +35,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; import org.fao.geonet.repository.MetadataValidationRepository; @@ -179,7 +179,7 @@ public void apply(@Nonnull AbstractMetadata entity) { dataMan.updateMetadata(context, id, processedMetadata, validate, ufo, language, new ISODate().toString(), updateDateStamp, IndexingMode.none); if (index) { - dataMan.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataMan.indexMetadata(id, DirectIndexSubmitter.INSTANCE); } } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index fef84962311..0ab8dfa62ea 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -80,8 +80,8 @@ import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.MetadataDraftRepository; @@ -243,7 +243,7 @@ public void deleteRecord( store.delResources(context, metadata.getUuid(), approved); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, metadata.getId() + ""); - metadataManager.deleteMetadata(context, metadata.getId() + "", DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(context, metadata.getId() + "", DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); } @@ -302,7 +302,7 @@ public SimpleMetadataProcessingReport deleteRecords( store.delResources(context, metadata.getUuid()); RecordDeletedEvent recordDeletedEvent = triggerDeletionEvent(request, String.valueOf(metadata.getId())); - metadataManager.deleteMetadata(context, String.valueOf(metadata.getId()), DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(context, String.valueOf(metadata.getId()), DirectDeletionSubmitter.INSTANCE); recordDeletedEvent.publish(ApplicationContextHolder.get()); report.incrementProcessedRecords(); @@ -1022,7 +1022,7 @@ private Pair loadRecord(MetadataType metadataType, Element xmlE metadataValidator.doValidate(metadata, context.getLanguage()); } - dataManager.indexMetadata(id.get(0), DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(id.get(0), DirectIndexSubmitter.INSTANCE); return Pair.read(Integer.valueOf(id.get(0)), uuid); } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java index 48b15b9a2c9..2cd757b9198 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSampleApi.java @@ -33,7 +33,6 @@ import jeeves.services.ReadWriteController; import org.apache.commons.lang.StringUtils; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.processing.report.SimpleMetadataProcessingReport; @@ -50,7 +49,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.mef.MEFLib; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.utils.Log; import org.fao.geonet.utils.Xml; @@ -291,7 +290,7 @@ SimpleMetadataProcessingReport addTemplates( if (MetadataType.lookup(isTemplate) == MetadataType.TEMPLATE) { xml = dataManager.setUUID(schemaName, uuid, xml); } - dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE); + dataManager.insertMetadata(context, metadata, xml, IndexingMode.full, true, UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE); report.addMetadataInfos(metadata, String.format( "Template for schema '%s' with UUID '%s' added.", diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java index 22c50ad1f55..d097928aa7f 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java @@ -55,7 +55,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.datamanager.*; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -819,7 +819,7 @@ public void setRecordGroup( metadata.getSourceInfo().setGroupOwner(groupIdentifier); metadataManager.save(metadata); - dataManager.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE); new RecordGroupOwnerChangeEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), @@ -1154,7 +1154,7 @@ private void checkCanPublishToAllGroup(ServiceContext context, ResourceBundle me if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - metadataIndexer.indexMetadata(metadata.getId() + "", DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(metadata.getId() + "", DirectIndexSubmitter.INSTANCE, IndexingMode.full); } boolean isInvalid = @@ -1218,7 +1218,7 @@ private void shareMetadataWithReservedGroup(String metadataUuid, boolean publish setOperations(sharing, dataManager, context, appContext, metadata, operationMap, privileges, ApiUtils.getUserSession(session).getUserIdAsInt(), true, null, request, metadataListToNotifyPublication, notifyByEmail); - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); java.util.Optional publicationOption = publicationConfig.getPublicationOptionConfiguration(publicationType); if (publicationOption.isPresent() && diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java index 0a56259a15e..c7239b9972d 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataUtils.java @@ -52,7 +52,7 @@ import org.fao.geonet.kernel.schema.AssociatedResourcesSchemaPlugin; import org.fao.geonet.kernel.schema.SchemaPlugin; import org.fao.geonet.kernel.search.EsSearchManager; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.lib.Lib; @@ -766,7 +766,7 @@ public static boolean retrieveMetadataValidationStatus(AbstractMetadata metadata if (!hasValidation) { validator.doValidate(metadata, context.getLanguage()); - dataManager.indexMetadata(metadata.getId() + "", DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(metadata.getId() + "", DirectIndexSubmitter.INSTANCE); } boolean isInvalid = diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java index e3e185e0418..6f9469cde66 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataValidateApi.java @@ -31,7 +31,6 @@ import jeeves.server.context.ServiceContext; import jeeves.services.ReadWriteController; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.records.editing.AjaxEditUtils; @@ -45,7 +44,7 @@ import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.GeonetworkDataDirectory; import org.fao.geonet.kernel.schema.MetadataSchema; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SchematronRepository; import org.fao.geonet.utils.IO; @@ -190,7 +189,7 @@ Reports validateRecord( .setStatus(isvalid ? MetadataValidationStatus.VALID : MetadataValidationStatus.INVALID) .setRequired(true).setNumTests(0).setNumFailures(0); this.metadataValidationRepository.save(metadataValidation); - dataManager.indexMetadata(("" + metadata.getId()), DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(("" + metadata.getId()), DirectIndexSubmitter.INSTANCE); new RecordValidationTriggeredEvent(metadata.getId(), ApiUtils.getUserSession(request.getSession()).getUserIdAsInt(), metadataValidation.getStatus().getCode()).publish(appContext); diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java index 1a14d3e07e0..5d32c63037d 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java @@ -58,7 +58,7 @@ import org.fao.geonet.kernel.metadata.StatusChangeType; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.languages.FeedbackLanguages; @@ -518,7 +518,7 @@ public Map setStatus(@Parameter(description = API_PAR if (statusUpdate.get(metadata.getId()) == StatusChangeType.UPDATED) { //--- reindex metadata - metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); // Reindex the metadata table record to update the field _statusWorkflow that contains the composite // status of the published and draft versions @@ -527,7 +527,7 @@ public Map setStatus(@Parameter(description = API_PAR if (metadataApproved != null) { metadataIdApproved = metadataApproved.getId(); - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } } @@ -946,7 +946,7 @@ public void restoreAtStatusSave( recoveredMetadataId = reloadRecord(element, metadataManager, httpSession, request); } - metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(recoveredMetadataId), DirectIndexSubmitter.INSTANCE, IndexingMode.full); UserSession session = ApiUtils.getUserSession(request.getSession()); if (session != null) { diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java index 33a4ddd3e71..18c1170a789 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java @@ -37,7 +37,7 @@ import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.schema.MetadataSchema; import org.fao.geonet.kernel.schema.SchemaPlugin; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.lib.Lib; import org.fao.geonet.schema.iso19139.ISO19139Namespaces; import org.fao.geonet.utils.Log; @@ -696,7 +696,7 @@ public synchronized boolean addAttribute(String id, String ref, String name, Str String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return true; } @@ -741,7 +741,7 @@ public synchronized boolean deleteAttribute(String id, String ref, String name, String changeDate = null; xmlSerializer.update(id, md, changeDate, false, null, context); - dataManager.indexMetadata(id, DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(id, DirectIndexSubmitter.INSTANCE); return true; } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java index 2f50c957bc1..7a0383f4d66 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java @@ -54,7 +54,7 @@ import org.fao.geonet.kernel.metadata.StatusActions; import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.*; @@ -486,7 +486,7 @@ public void saveEdits( if (reindex) { Log.trace(Geonet.DATA_MANAGER, " > Reindexing record"); - metadataIndexer.indexMetadata(id, DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(id, DirectIndexSubmitter.INSTANCE, IndexingMode.full); } // Reindex the metadata table record to update the field _statusWorkflow that contains the composite @@ -495,7 +495,7 @@ public void saveEdits( Metadata metadataApproved = metadataRepository.findOneByUuid(metadata.getUuid()); if (metadataApproved != null) { - metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(String.valueOf(metadataApproved.getId()), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } } diff --git a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java index 873fb2274c2..96e7c6a1225 100644 --- a/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java +++ b/services/src/main/java/org/fao/geonet/api/registries/DirectoryUtils.java @@ -3,7 +3,6 @@ import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; import com.google.common.collect.Table; -import jeeves.server.ServiceConfig; import jeeves.server.context.ServiceContext; import jeeves.xlink.XLink; import org.apache.commons.lang.StringUtils; @@ -13,7 +12,7 @@ import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.util.Sha1Encoder; import org.fao.geonet.utils.Log; @@ -74,7 +73,7 @@ public static Map saveEntries(ServiceContext context, (Element) entry.clone(), IndexingMode.none, ufo, UpdateDatestamp.NO, - publicForGroup, DirectIndexSubmittor.INSTANCE); + publicForGroup, DirectIndexSubmitter.INSTANCE); collectResults.getEntryIdentifiers().put( uuid, subtemplate.getId()); diff --git a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java index d2ade99f3bf..6932db2edd8 100644 --- a/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java +++ b/services/src/main/java/org/fao/geonet/api/selections/UserSelectionsApi.java @@ -37,8 +37,8 @@ import org.fao.geonet.kernel.datamanager.IMetadataIndexer; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmittor; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.batch.BatchingIndexSubmitter; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.LanguageRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.repository.UserGroupRepository; @@ -327,7 +327,7 @@ ResponseEntity addToUserSelection( UserSavedSelection e = new UserSavedSelection(selection.get(), user.get(), u); try { umsRepository.save(e); - metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), DirectIndexSubmittor.INSTANCE, IndexingMode.full); + metadataIndexer.indexMetadata(metadataRepository.getMetadataId(u), DirectIndexSubmitter.INSTANCE, IndexingMode.full); } catch (Exception e1) { Log.error(API.LOG_MODULE_NAME, "UserSelectionsApi - addToUserSelection: " + e1.getMessage(), e1); } @@ -386,7 +386,7 @@ ResponseEntity deleteFromUserSelection( if (uuid == null || uuid.length == 0) { umsRepository.deleteAllBySelectionAndUser(selectionIdentifier, userIdentifier); } else { - try (BatchingIndexSubmittor batchingIndexSubmittor = new BatchingIndexSubmittor(uuid.length)) { + try (BatchingIndexSubmitter batchingIndexSubmittor = new BatchingIndexSubmitter(uuid.length)) { for (String u : uuid) { UserSavedSelectionId e = new UserSavedSelectionId() .setSelectionId(selectionIdentifier) diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java index 297f181abc7..2b558dd0295 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/InspireValidationRunnable.java @@ -26,18 +26,14 @@ import jeeves.transaction.TransactionManager; import jeeves.transaction.TransactionTask; import org.fao.geonet.ApplicationContextHolder; -import org.fao.geonet.inspire.validator.InspireValidatorUtils; import org.fao.geonet.domain.MetadataValidation; import org.fao.geonet.domain.MetadataValidationId; import org.fao.geonet.domain.MetadataValidationStatus; import org.fao.geonet.kernel.DataManager; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataValidationRepository; import org.springframework.transaction.TransactionStatus; -import java.util.ArrayList; -import java.util.Arrays; - import static jeeves.transaction.TransactionManager.CommitBehavior.ALWAYS_COMMIT; import static jeeves.transaction.TransactionManager.TransactionRequirement.CREATE_NEW; @@ -92,7 +88,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { DataManager dataManager = ApplicationContextHolder.get().getBean(DataManager.class); - dataManager.indexMetadata(mdId + "", DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(mdId + "", DirectIndexSubmitter.INSTANCE); return null; } diff --git a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java index a74d53f2a1c..61036c007c8 100644 --- a/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java +++ b/services/src/main/java/org/fao/geonet/inspire/validator/MInspireEtfValidateProcess.java @@ -16,7 +16,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.datamanager.IMetadataSchemaUtils; import org.fao.geonet.kernel.datamanager.IMetadataUtils; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -34,8 +34,6 @@ import javax.management.ObjectName; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -264,7 +262,7 @@ public Object doInTransaction(TransactionStatus transaction) throws Throwable { } if (reindexMetadata) { - dataManager.indexMetadata(metadataRecord.getId() + "", DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata(metadataRecord.getId() + "", DirectIndexSubmitter.INSTANCE); } } catch (Exception ex) { diff --git a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java index d22e9da3039..b24fb3bbfb4 100644 --- a/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/MetadataValidateApiTest.java @@ -24,7 +24,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.SourceRepository; @@ -209,9 +209,9 @@ private AbstractMetadata subTemplateOnLineResourceDbInsert(MetadataType type) th false, NO, false, - DirectIndexSubmittor.INSTANCE); + DirectIndexSubmitter.INSTANCE); - dataManager.indexMetadata("" + dbInsertedMetadata.getId(), DirectIndexSubmittor.INSTANCE); + dataManager.indexMetadata("" + dbInsertedMetadata.getId(), DirectIndexSubmitter.INSTANCE); assertEquals(1, countTemplateIndexed(dbInsertedMetadata.getUuid(), false, type == MetadataType.SUB_TEMPLATE ? "s" : "n")); return dbInsertedMetadata; } diff --git a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java index 462efae380e..641f477a99f 100644 --- a/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/extent/MetadataExtentApiTest.java @@ -24,7 +24,6 @@ package org.fao.geonet.api.records.extent; import jeeves.server.context.ServiceContext; -import org.fao.geonet.domain.AbstractMetadata; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Metadata; import org.fao.geonet.domain.MetadataType; @@ -32,7 +31,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.repository.SourceRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.fao.geonet.utils.Xml; @@ -56,14 +55,11 @@ import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.Arrays; import java.util.Calendar; import java.util.GregorianCalendar; import java.util.TimeZone; import java.util.UUID; -import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO; -import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; @@ -340,7 +336,7 @@ private String createMdFromXmlRessources(Element sampleMetadataXml) throws Excep .setHarvested(false); dataManager.insertMetadata(context, metadata, sampleMetadataXml, IndexingMode.full, false, UpdateDatestamp.NO, - false, DirectIndexSubmittor.INSTANCE).getId(); + false, DirectIndexSubmitter.INSTANCE).getId(); return uuid; } diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java index 0d62fd75be9..41e96825bc5 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/AbstractFormatterTest.java @@ -34,7 +34,7 @@ import org.fao.geonet.kernel.SchemaManager; import org.fao.geonet.kernel.UpdateDatestamp; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.languages.IsoLanguagesMapper; import org.fao.geonet.repository.MetadataRepository; @@ -98,7 +98,7 @@ public void setUp() throws Exception { metadata.setHarvestInfo(harvestInfo); metadata.setData(xml); this.id = dataManager.insertMetadata(serviceContext, metadata, metadata.getXmlData(false), IndexingMode.full, false, - UpdateDatestamp.NO, false, DirectIndexSubmittor.INSTANCE).getId(); + UpdateDatestamp.NO, false, DirectIndexSubmitter.INSTANCE).getId(); } public boolean isHarvested() { diff --git a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java index aba2e64c270..aded0661464 100644 --- a/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java +++ b/services/src/test/java/org/fao/geonet/api/records/formatters/cache/FormatterCacheIntegrationTest.java @@ -32,7 +32,7 @@ import org.fao.geonet.domain.ReservedOperation; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest.ImportMetadata; -import org.fao.geonet.kernel.search.submission.DirectDeletionSubmittor; +import org.fao.geonet.kernel.search.submission.DirectDeletionSubmitter; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.OperationAllowedSpecs; import org.fao.geonet.services.AbstractServiceIntegrationTest; @@ -135,7 +135,7 @@ public void testUpdatesAfterMetadataDeleted() throws Exception { final Key key = new Key(Integer.parseInt(metadataId), "eng", FormatType.html, "full_view", true, FormatterWidth._100); formatterCache.get(key, new ChangeDateValidator(changeDate), new TestLoader("result", changeDate, true), true); - metadataManager.deleteMetadata(createServiceContext(), metadataId, DirectDeletionSubmittor.INSTANCE); + metadataManager.deleteMetadata(createServiceContext(), metadataId, DirectDeletionSubmitter.INSTANCE); entityManager.flush(); assertNull(formatterCache.getPublished(key)); diff --git a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java index afa379d5752..6abc191bbb1 100644 --- a/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java +++ b/services/src/test/java/org/fao/geonet/api/selections/UserSelectionsApiTest.java @@ -32,15 +32,14 @@ import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.DirectIndexSubmitter; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.SelectionRepository; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.Spy; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpSession; @@ -230,7 +229,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isCreated()); - verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), eq(DirectIndexSubmittor.INSTANCE), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(1)).indexMetadata(eq(metadataId), eq(DirectIndexSubmitter.INSTANCE), eq(IndexingMode.full)); this.mockMvc.perform(get("/srv/api/userselections/" + createdSelection.getId() + "/1") .session(this.mockHttpSession) @@ -245,7 +244,7 @@ public void addDeleteSelection() throws Exception { .accept(MediaType.parseMediaType("application/json"))) .andExpect(status().isNoContent()); - verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(IIndexSubmittor.class), eq(IndexingMode.full)); + verify(this.metadataIndexerSpy, times(2)).indexMetadata(eq(metadataId), any(IIndexSubmitter.class), eq(IndexingMode.full)); // Delete this.mockMvc.perform(delete("/srv/api/userselections/" + createdSelection.getId()) diff --git a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java index 293dd9c6f72..9926a905467 100644 --- a/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java +++ b/services/src/test/java/org/fao/geonet/services/metadata/BatchOpsMetadatReindexerTest.java @@ -5,8 +5,7 @@ import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.ISearchManager; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; -import org.fao.geonet.kernel.search.submission.DirectIndexSubmittor; -import org.fao.geonet.kernel.search.submission.IIndexSubmittor; +import org.fao.geonet.kernel.search.submission.IIndexSubmitter; import org.fao.geonet.util.ThreadUtils; import org.junit.Test; import org.junit.runner.RunWith; @@ -194,7 +193,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { usedTread.add(Thread.currentThread()); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmittor.class)); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmitter.class)); return mockDataMan; } @@ -208,7 +207,7 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable { latch.await(); return null; } - }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmittor.class)); + }).when(mockDataMan).indexMetadata(Mockito.anyString(), Mockito.any(IIndexSubmitter.class)); return mockDataMan; } @@ -223,7 +222,7 @@ private Set createMetadataToIndex() { private ArgumentCaptor captureIndexationLaunched(DataManager mockDataMan) throws Exception { ArgumentCaptor metadataIdCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor indexSubmittorCaptor = ArgumentCaptor.forClass(IIndexSubmittor.class); + ArgumentCaptor indexSubmittorCaptor = ArgumentCaptor.forClass(IIndexSubmitter.class); ArgumentCaptor isearchManagerCaptor = ArgumentCaptor.forClass(ISearchManager.class); Mockito.verify(mockDataMan, Mockito.times(4)).indexMetadata(metadataIdCaptor.capture(), indexSubmittorCaptor.capture()); From 2b6476cd932cf9ca3a5b1c57c6bd46c3d56b5bc8 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Fri, 6 Dec 2024 16:30:43 +0100 Subject: [PATCH 16/18] Remove debug sleep --- .../java/org/fao/geonet/kernel/DataManagerIntegrationTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java index 52120b59dac..ee046f77c92 100644 --- a/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/DataManagerIntegrationTest.java @@ -184,8 +184,6 @@ public void testDeleteBatchMetadata() throws Exception { dataManager.batchDeleteMetadataAndUpdateIndex(spec); - Thread.sleep(500); - assertEquals(0, metadataRepository.findAll(spec).size()); assertEquals(0, searchManager.query(String.format("uuid:(%s OR %s)", uuid1, uuid2), null, 0, 10).hits().hits().size()); } From b8952a04a79ff438ae4d6136aa3505853065cae4 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Fri, 6 Dec 2024 16:33:31 +0100 Subject: [PATCH 17/18] Remove unused never implemented that still references forceRefreshReaders --- .../fao/geonet/index/es/IndexationTest.java | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java diff --git a/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java b/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java deleted file mode 100644 index 38b41744168..00000000000 --- a/estest/src/test/java/org/fao/geonet/index/es/IndexationTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.fao.geonet.index.es; - -import org.fao.geonet.domain.MetadataType; -import org.fao.geonet.kernel.search.EsSearchManager; -import org.jdom.Element; -import org.junit.Ignore; -import org.junit.Test; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; - -public class IndexationTest { - - @Ignore - @Test - public void nominal() throws Exception { - - Path schemaDir = Paths.get(this.getClass().getResource("/nominal").toURI()); - Element metadata = new Element("root"); - String id = "test-id"; - Map moreFields = new HashMap<>(); - MetadataType metadataType = MetadataType.METADATA; - String root = "dummy"; - boolean forceRefreshReaders = true; - - EsSearchManager toTest = new EsSearchManager(); - -// toTest.index(schemaDir, metadata, id, moreFields, metadataType, root, forceRefreshReaders); -// -// String docToBeIndexed = toTest.listOfDocumentsToIndex.get(id); -// Map result = new ObjectMapper().readValue(docToBeIndexed, HashMap.class); -// Assert.assertEquals(3, result.size()); -// Assert.assertEquals("test-id", result.get("id")); -// Assert.assertEquals("metadata", result.get("docType")); -// Assert.assertEquals("source-from-index-xsl", result.get("sourceCatalogue")); - } -} From d6b9655ef517fbbe0403f1fda71a096b20f96675 Mon Sep 17 00:00:00 2001 From: Tobias Hotz Date: Fri, 6 Dec 2024 16:38:55 +0100 Subject: [PATCH 18/18] Improve log message when deleting --- .../fao/geonet/kernel/datamanager/base/BaseMetadataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index 0554b9b3bd7..405c47e2cc8 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -254,8 +254,8 @@ public void synchronizeDbWithIndex(ServiceContext context) throws Exception { // remove from index metadata not in DBMS try (BatchingDeletionSubmitter submitter = new BatchingDeletionSubmitter()) { for (String id : docs.keySet()) { + LOGGER_DATA_MANAGER.debug("- removing record ({}) from index", id); getSearchManager().deleteByQuery(String.format("+id:%s", id), submitter); - LOGGER_DATA_MANAGER.debug("- removed record ({}) from index", id); } } }