From 35096c7a1c437fd2c2b548fd3974d89d11eaa6b9 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 28 Jun 2024 16:25:32 +0000 Subject: [PATCH] fixes #2452: Added a local ivarator cache dir configuration --- .../input/microservice/QueryLogicFactory.xml | 4 ++++ .../input/webservice/QueryLogicFactory.xml | 6 ++++++ .../webservice/properties/default.properties | 4 ++++ .../rendered/microservice/QueryLogicFactory.xml | 4 ++++ .../rendered/webservice/QueryLogicFactory.xml | 8 ++++++++ microservices/starters/query | 2 +- properties/bamboo.properties | 1 + properties/default.properties | 4 ++++ .../query/config/ShardQueryConfiguration.java | 17 +++++++++++++++++ .../datawave/query/iterator/QueryIterator.java | 2 +- .../datawave/query/tables/ShardQueryLogic.java | 10 +++++++++- .../query/transformer/UniqueTransform.java | 2 +- .../config/ShardQueryConfigurationTest.java | 2 ++ .../UniqueTransformMostRecentTest.java | 3 ++- .../datawave/query/EventQueryLogicFactory.xml | 1 + .../datawave/query/QueryLogicFactory.xml | 1 + .../datawave/query/QueryLogicFactory.xml | 6 ++++++ 17 files changed, 72 insertions(+), 5 deletions(-) diff --git a/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml index e6233a4fac1..49e272ecc5f 100644 --- a/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml +++ b/microservices/configcheck/src/test/resources/input/microservice/QueryLogicFactory.xml @@ -114,6 +114,10 @@ + + + + diff --git a/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml index 8c40461dcdf..f7d81628ee7 100644 --- a/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml +++ b/microservices/configcheck/src/test/resources/input/webservice/QueryLogicFactory.xml @@ -284,6 +284,8 @@ + + @@ -324,6 +326,10 @@ + + ${local.ivarator.cache.dir.config} + + ${ivarator.cache.dir.config} diff --git a/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties b/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties index 2adeacbcc0e..e76380a8030 100644 --- a/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties +++ b/microservices/configcheck/src/test/resources/input/webservice/properties/default.properties @@ -451,6 +451,10 @@ hdfs.site.config.urls=file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/c ## - minAvailableStoragePercent - (Optional, Default: 0.0) A double, between 0.0 and 1.0, which specifies the ## minimum percent of available storage space required to persist to this ivarator path. If less than this ## percent is available, we will not use this ivarator path. +local.ivarator.cache.dir.config= \ + \ +\n \ +\n ivarator.cache.dir.config= \ \ \n \ diff --git a/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml index bb07d23456d..8de5a940ece 100644 --- a/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml +++ b/microservices/configcheck/src/test/resources/rendered/microservice/QueryLogicFactory.xml @@ -114,6 +114,10 @@ + + + + diff --git a/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml b/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml index 558d4b05073..595e52252fe 100644 --- a/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml +++ b/microservices/configcheck/src/test/resources/rendered/webservice/QueryLogicFactory.xml @@ -276,6 +276,8 @@ + + @@ -316,6 +318,12 @@ + + + + + + diff --git a/microservices/starters/query b/microservices/starters/query index 165530a2b38..368802eee72 160000 --- a/microservices/starters/query +++ b/microservices/starters/query @@ -1 +1 @@ -Subproject commit 165530a2b3802a696ec665412516d9d43b850ba7 +Subproject commit 368802eee7271080798af60c05160a032a85484e diff --git a/properties/bamboo.properties b/properties/bamboo.properties index 16a50283f28..961ffa466af 100644 --- a/properties/bamboo.properties +++ b/properties/bamboo.properties @@ -1,4 +1,5 @@ #This is for the unit testing of spring configuration files by bamboo accumulo.user.password=testdummy +local.ivarator.cache.dir.config= ivarator.cache.dir.config= ivarator.fst.hdfs.base.uris= diff --git a/properties/default.properties b/properties/default.properties index c4ca9644aab..b7630b05cd8 100644 --- a/properties/default.properties +++ b/properties/default.properties @@ -497,6 +497,10 @@ hdfs.site.config.urls=file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/c ## - minAvailableStoragePercent - (Optional, Default: 0.0) A double, between 0.0 and 1.0, which specifies the ## minimum percent of available storage space required to persist to this ivarator path. If less than this ## percent is available, we will not use this ivarator path. +local.ivarator.cache.dir.config= \ + \ +\n \ +\n ivarator.cache.dir.config= \ \ \n \ diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java index 2dbfdf655b4..f8a41fed912 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java @@ -351,7 +351,10 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement private String hdfsSiteConfigURLs = null; private String hdfsFileCompressionCodec = null; private String zookeeperConfig = null; + // tserver side ivarator cache dir configs private List ivaratorCacheDirConfigs = Collections.emptyList(); + // webserver side ivarator cache dir configs (e.g. for unique transform on webserver) + private List localIvaratorCacheDirConfigs = Collections.emptyList(); private String ivaratorFstHdfsBaseURIs = null; private int ivaratorCacheBufferSize = 10000; @@ -655,6 +658,8 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) { this.setHdfsSiteConfigURLs(other.getHdfsSiteConfigURLs()); this.setHdfsFileCompressionCodec(other.getHdfsFileCompressionCodec()); this.setZookeeperConfig(other.getZookeeperConfig()); + this.setLocalIvaratorCacheDirConfigs( + null == other.getLocalIvaratorCacheDirConfigs() ? null : Lists.newArrayList(other.getLocalIvaratorCacheDirConfigs())); this.setIvaratorCacheDirConfigs(null == other.getIvaratorCacheDirConfigs() ? null : Lists.newArrayList(other.getIvaratorCacheDirConfigs())); this.setIvaratorFstHdfsBaseURIs(other.getIvaratorFstHdfsBaseURIs()); this.setIvaratorCacheBufferSize(other.getIvaratorCacheBufferSize()); @@ -749,6 +754,8 @@ public ShardQueryConfiguration(ShardQueryConfiguration other, Collection ivaratorCach this.ivaratorCacheDirConfigs = ivaratorCacheDirConfigs; } + public void setLocalIvaratorCacheDirConfigs(List localIvaratorCacheDirConfigs) { + this.localIvaratorCacheDirConfigs = localIvaratorCacheDirConfigs; + } + + public List getLocalIvaratorCacheDirConfigs() { + return localIvaratorCacheDirConfigs; + } + public String getIvaratorFstHdfsBaseURIs() { return ivaratorFstHdfsBaseURIs; } @@ -2918,6 +2933,7 @@ public boolean equals(Object o) { Objects.equals(getHdfsSiteConfigURLs(), that.getHdfsSiteConfigURLs()) && Objects.equals(getHdfsFileCompressionCodec(), that.getHdfsFileCompressionCodec()) && Objects.equals(getZookeeperConfig(), that.getZookeeperConfig()) && + Objects.equals(getLocalIvaratorCacheDirConfigs(), that.getLocalIvaratorCacheDirConfigs()) && Objects.equals(getIvaratorCacheDirConfigs(), that.getIvaratorCacheDirConfigs()) && Objects.equals(getIvaratorFstHdfsBaseURIs(), that.getIvaratorFstHdfsBaseURIs()) && Objects.equals(getQueryModel(), that.getQueryModel()) && @@ -3097,6 +3113,7 @@ public int hashCode() { getHdfsSiteConfigURLs(), getHdfsFileCompressionCodec(), getZookeeperConfig(), + getLocalIvaratorCacheDirConfigs(), getIvaratorCacheDirConfigs(), getIvaratorFstHdfsBaseURIs(), getIvaratorCacheBufferSize(), diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java index 23954939d0c..fc5d367269d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java @@ -1548,7 +1548,7 @@ protected UniqueTransform getUniqueTransform() throws IOException { .withUniqueFields(getUniqueFields()) .withQueryExecutionForPageTimeout(getResultTimeout()) .withBufferPersistThreshold(getUniqueCacheBufferSize()) - .withIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs()) + .withLocalIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs()) .withHdfsSiteConfigURLs(getHdfsSiteConfigURLs()) .withSubDirectory(getQueryId() + "-" + getScanId()) .withMaxOpenFiles(getIvaratorMaxOpenFiles()) diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 4ce5f1c1b8e..e798b1a7f21 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -677,7 +677,7 @@ private void addConfigBasedTransformers() throws QueryException { .withQueryExecutionForPageTimeout(this.getQueryExecutionForPageTimeout()) .withModel(getQueryModel()) .withBufferPersistThreshold(getUniqueCacheBufferSize()) - .withIvaratorCacheDirConfigs(getIvaratorCacheDirConfigs()) + .withLocalIvaratorCacheDirConfigs(getLocalIvaratorCacheDirConfigs()) .withHdfsSiteConfigURLs(getHdfsSiteConfigURLs()) .withSubDirectory(getConfig().getQuery().getId().toString()) .withMaxOpenFiles(getIvaratorMaxOpenFiles()) @@ -1926,6 +1926,14 @@ public void setFullTableScanEnabled(boolean fullTableScanEnabled) { getConfig().setFullTableScanEnabled(fullTableScanEnabled); } + public void setLocalIvaratorCacheDirConfigs(List localIvaratorCacheDirConfigs) { + getConfig().setLocalIvaratorCacheDirConfigs(localIvaratorCacheDirConfigs); + } + + public List getLocalIvaratorCacheDirConfigs() { + return getConfig().getLocalIvaratorCacheDirConfigs(); + } + public List getIvaratorCacheDirConfigs() { return getConfig().getIvaratorCacheDirConfigs(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java index d68ae473dbc..cf46ccee418 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java @@ -488,7 +488,7 @@ public Builder withBufferPersistThreshold(int bufferPersistThreshold) { return this; } - public Builder withIvaratorCacheDirConfigs(List ivaratorCacheDirConfigs) { + public Builder withLocalIvaratorCacheDirConfigs(List ivaratorCacheDirConfigs) { this.ivaratorCacheDirConfigs = ivaratorCacheDirConfigs; return this; } diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java index 895a2cbae9f..432b05a3fa1 100644 --- a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java @@ -389,6 +389,8 @@ public void setUp() throws Exception { updatedValues.put("hdfsFileCompressionCodec", "sunny"); defaultValues.put("zookeeperConfig", null); updatedValues.put("zookeeperConfig", "file://etc/zookeeper/conf"); + defaultValues.put("localIvaratorCacheDirConfigs", Collections.emptyList()); + updatedValues.put("localIvaratorCacheDirConfigs", Lists.newArrayList(new IvaratorCacheDirConfig("file:///tmp/ivarators"))); defaultValues.put("ivaratorCacheDirConfigs", Collections.emptyList()); updatedValues.put("ivaratorCacheDirConfigs", Lists.newArrayList(new IvaratorCacheDirConfig("hdfs://instance-a/ivarators"))); defaultValues.put("ivaratorFstHdfsBaseURIs", null); diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java b/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java index 5487ae7162c..14e52cb46ad 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/UniqueTransformMostRecentTest.java @@ -35,6 +35,7 @@ public void setUp() throws IOException { // setup a directory for cache results File tmpDir = temporaryFolder.newFolder(); IvaratorCacheDirConfig config = new IvaratorCacheDirConfig(tmpDir.toURI().toString()); + logic.setLocalIvaratorCacheDirConfigs(Collections.singletonList(config)); logic.setIvaratorCacheDirConfigs(Collections.singletonList(config)); QueryImpl query = new QueryImpl(); @@ -50,7 +51,7 @@ protected UniqueTransform getUniqueTransform() { .withUniqueFields(uniqueFields) .withQueryExecutionForPageTimeout(Long.MAX_VALUE) .withBufferPersistThreshold(logic.getUniqueCacheBufferSize()) - .withIvaratorCacheDirConfigs(logic.getIvaratorCacheDirConfigs()) + .withLocalIvaratorCacheDirConfigs(logic.getLocalIvaratorCacheDirConfigs()) .withHdfsSiteConfigURLs(logic.getHdfsSiteConfigURLs()) .withSubDirectory(logic.getConfig().getQuery().getId().toString()) .withMaxOpenFiles(logic.getIvaratorMaxOpenFiles()) diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml index ffa8088b8e0..3f85bd48e87 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml @@ -114,6 +114,7 @@ + diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml index fe08b21c663..9a823e57144 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml @@ -232,6 +232,7 @@ + diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml index 5adce1647d0..418458a4154 100644 --- a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml +++ b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml @@ -285,6 +285,8 @@ + + @@ -325,6 +327,10 @@ + + ${local.ivarator.cache.dir.config} + + ${ivarator.cache.dir.config}