Optionally build a precise datatype filter when no filter is specified (

NationalSecurityAgency#2333)
cawaring · Apr 9, 2024 · 6730346 · 6730346
1 parent fb5b0f1
commit 6730346
Show file tree

Hide file tree

Showing 9 changed files with 99 additions and 17 deletions.
diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -100,6 +100,12 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
     private int collapseUidsThreshold = -1;
     // Should this query dedupe terms within ANDs and ORs
     private boolean enforceUniqueTermsWithinExpressions = false;
+    // After query planning rebuild the datatype filter from the remaining query fields.
+    // The actual filter may be a subset of the requested datatypes. This has implications
+    // for the global index lookup and execution of regex terms.
+    private boolean rebuildDatatypeFilter = false;
+    private boolean rebuildDatatypeFilterPerShard = false;
+    // reduces the datatype filter, respecting the user-supplied datatypes
     private boolean reduceIngestTypes = false;
     private boolean reduceIngestTypesPerShard = false;
     // should this query attempt to prune terms via their ingest types
@@ -520,6 +526,8 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
         this.setReduceQueryFieldsPerShard(other.getReduceQueryFieldsPerShard());
         this.setReduceTypeMetadata(other.getReduceTypeMetadata());
         this.setReduceTypeMetadataPerShard(other.getReduceTypeMetadataPerShard());
+        this.setRebuildDatatypeFilter(other.isRebuildDatatypeFilter());
+        this.setRebuildDatatypeFilterPerShard(other.isRebuildDatatypeFilterPerShard());
         this.setParseTldUids(other.getParseTldUids());
         this.setSequentialScheduler(other.getSequentialScheduler());
         this.setCollectTimingDetails(other.getCollectTimingDetails());
@@ -2627,6 +2635,22 @@ public void setPruneQueryOptions(boolean pruneQueryOptions) {
         this.pruneQueryOptions = pruneQueryOptions;
     }
 
+    public boolean isRebuildDatatypeFilter() {
+        return rebuildDatatypeFilter;
+    }
+
+    public void setRebuildDatatypeFilter(boolean rebuildDatatypeFilter) {
+        this.rebuildDatatypeFilter = rebuildDatatypeFilter;
+    }
+
+    public boolean isRebuildDatatypeFilterPerShard() {
+        return rebuildDatatypeFilterPerShard;
+    }
+
+    public void setRebuildDatatypeFilterPerShard(boolean rebuildDatatypeFilterPerShard) {
+        this.rebuildDatatypeFilterPerShard = rebuildDatatypeFilterPerShard;
+    }
+
     public boolean getReduceIngestTypes() {
         return reduceIngestTypes;
     }

diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java
@@ -29,8 +29,6 @@
 import org.apache.commons.jexl3.parser.JexlNode;
 import org.apache.log4j.Logger;
 
-import com.google.common.collect.Sets;
-
 import datawave.query.jexl.JexlASTHelper;
 import datawave.query.jexl.functions.ContentFunctionsDescriptor;
 import datawave.query.jexl.functions.EvaluationPhaseFilterFunctionsDescriptor;
@@ -45,9 +43,15 @@
 import datawave.query.util.TypeMetadata;
 
 /**
- * Visitor that returns the set of all ingest types associated with an arbitrary JexlNode.
+ * Visitor that returns the <b>effective set</b> of ingest types associated with an arbitrary JexlNode.
+ * <p>
+ * The effective set is calculated by applying union and intersection logic to produce the reduced set of ingest types for complex tree structures. For example:
+ * <p>
+ * <code>(A AND B)</code> when the A term maps to ingest type 1 and the B term maps to ingest types 1, 2, and 3.
  * <p>
- * Much of this code is duplicated from the {@link IngestTypePruningVisitor}.
+ * The full set of ingest types is {1, 2, 3}, but the <b>effective set</b> is just ingest type 1.
+ * <p>
+ * Much of this code is originated from the {@link IngestTypePruningVisitor}.
  */
 public class IngestTypeVisitor extends BaseVisitor {
 
@@ -262,7 +266,7 @@ public Set<String> getIngestTypesForJunction(JexlNode node) {
     public Set<String> getIngestTypesForLeaf(JexlNode node) {
         node = JexlASTHelper.dereference(node);
         if (node instanceof ASTEQNode) {
-            Object literal = JexlASTHelper.getLiteralValue(node);
+            Object literal = JexlASTHelper.getLiteralValueSafely(node);
             if (literal == null) {
                 return Collections.singleton(UNKNOWN_TYPE);
             }
@@ -356,6 +360,10 @@ private Set<String> getIngestTypesForIntersection(ASTAndNode node) {
             JexlNode child = JexlASTHelper.dereference(node.jjtGetChild(i));
             Set<String> childIngestTypes = (Set<String>) child.jjtAccept(this, null);
 
+            if (childIngestTypes == null) {
+                continue; // we could have a malformed query or a query with a _Drop_ marker
+            }
+
             if (ingestTypes.isEmpty()) {
                 ingestTypes = childIngestTypes;
             } else {
@@ -379,6 +387,7 @@ private Set<String> intersectTypes(Set<String> typesA, Set<String> typesB) {
         if (typesA.contains(UNKNOWN_TYPE) || typesB.contains(UNKNOWN_TYPE)) {
             return Collections.singleton(UNKNOWN_TYPE);
         }
-        return Sets.intersection(typesA, typesB);
+        typesA.retainAll(typesB);
+        return typesA;
     }
 }
diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
@@ -2220,6 +2220,11 @@ protected Future<IteratorSetting> loadQueryIterator(final MetadataHelper metadat
                 throw new DatawaveQueryException(qe);
             }
 
+            if (!preloadOptions && config.isRebuildDatatypeFilter()) {
+                Set<String> datatypes = IngestTypeVisitor.getIngestTypes(config.getQueryTree(), getTypeMetadata());
+                config.setDatatypeFilter(datatypes);
+            }
+
             String datatypeFilter = config.getDatatypeFilterAsString();
 
             addOption(cfg, QueryOptions.DATATYPE_FILTER, datatypeFilter, false);
@@ -2652,6 +2657,12 @@ public Tuple2<CloseableIterable<QueryPlan>,Boolean> getQueryRanges(ScannerFactor
             fullTableScanReason = state.reason;
         }
 
+        // optionally build/rebuild the datatype filter with the fully planned query
+        if (config.isRebuildDatatypeFilter()) {
+            Set<String> ingestTypes = IngestTypeVisitor.getIngestTypes(config.getQueryTree(), getTypeMetadata());
+            config.setDatatypeFilter(ingestTypes);
+        }
+
         Set<String> ingestTypes = null;
         if (config.getReduceIngestTypes()) {
             Set<String> userRequestedIngestTypes = config.getDatatypeFilter();

diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -2827,4 +2827,20 @@ public boolean getSortQueryByCounts() {
     public void setSortQueryByCounts(boolean sortQueryByCounts) {
         getConfig().setSortQueryByCounts(sortQueryByCounts);
     }
+
+    public boolean isRebuildDatatypeFilter() {
+        return getConfig().isRebuildDatatypeFilter();
+    }
+
+    public void setRebuildDatatypeFilter(boolean rebuildDatatypeFilter) {
+        getConfig().setRebuildDatatypeFilter(rebuildDatatypeFilter);
+    }
+
+    public boolean isRebuildDatatypeFilterPerShard() {
+        return getConfig().isRebuildDatatypeFilterPerShard();
+    }
+
+    public void setRebuildDatatypeFilterPerShard(boolean rebuildDatatypeFilterPerShard) {
+        getConfig().setRebuildDatatypeFilterPerShard(rebuildDatatypeFilterPerShard);
+    }
 }
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java b/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
@@ -7,6 +7,7 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -28,9 +29,11 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
+import org.geotools.data.Join;
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Sets;
@@ -313,7 +316,7 @@ public ScannerChunk apply(@Nullable ScannerChunk input) {
                         reduceQueryFields(script, newIteratorSetting);
                     }
 
-                    if (config.getReduceIngestTypesPerShard()) {
+                    if (config.isRebuildDatatypeFilterPerShard() || config.getReduceIngestTypesPerShard()) {
                         reduceIngestTypes(script, newIteratorSetting);
                     }
 
@@ -491,17 +494,29 @@ private void reduceIngestTypes(ASTJexlScript script, IteratorSetting newIterator
             cachedTypeMetadata = new TypeMetadata(serializedTypeMetadata);
         }
 
-        Set<String> userRequestedDataTypes = config.getDatatypeFilter();
-        if (!userRequestedDataTypes.isEmpty()) {
-            Set<String> queryDataTypes = IngestTypeVisitor.getIngestTypes(script, cachedTypeMetadata);
-            Set<String> ingestTypes = Sets.intersection(userRequestedDataTypes, queryDataTypes);
-            if (ingestTypes.size() < userRequestedDataTypes.size()) {
-                newIteratorSetting.addOption(QueryOptions.DATATYPE_FILTER, Joiner.on(',').join(ingestTypes));
-            }
+        // get requested types
+        Set<String> requestedDatatypes;
+        String opt = newIteratorSetting.getOptions().get(QueryOptions.DATATYPE_FILTER);
+        if (opt == null) {
+            requestedDatatypes = Collections.emptySet();
+        } else {
+            requestedDatatypes = new HashSet<>(Splitter.on(',').splitToList(opt));
+        }
 
-            if (ingestTypes.isEmpty()) {
+        // get existing types from the query
+        Set<String> datatypes = IngestTypeVisitor.getIngestTypes(script, cachedTypeMetadata);
+
+        if (config.isRebuildDatatypeFilterPerShard()) {
+            newIteratorSetting.addOption(QueryOptions.DATATYPE_FILTER, Joiner.on(',').join(datatypes));
+        } else if (config.getReduceIngestTypesPerShard() && !requestedDatatypes.isEmpty() && !datatypes.isEmpty()) {
+            Set<String> intersectedTypes = Sets.intersection(requestedDatatypes, datatypes);
+            if (intersectedTypes.isEmpty()) {
                 // the EmptyPlanPruner in the RangeStream should have handled this situation, this exception indicates a bug exists
-                throw new DatawaveFatalQueryException("Reduced ingest types to zero, cannot execute query sub-plan");
+                throw new DatawaveFatalQueryException("Ingest types reduced to zero, cannot execute query sub-plan");
+            }
+
+            if (intersectedTypes.size() <= requestedDatatypes.size()) {
+                newIteratorSetting.addOption(QueryOptions.DATATYPE_FILTER, Joiner.on(',').join(intersectedTypes));
             }
         }
     }

diff --git a/warehouse/query-core/src/test/java/datawave/query/LenientFieldsTest.java b/warehouse/query-core/src/test/java/datawave/query/LenientFieldsTest.java
@@ -20,7 +20,6 @@
 import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Key;

diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
@@ -137,6 +137,10 @@ public void setUp() throws Exception {
         updatedValues.put("reduceQueryFields", true);
         defaultValues.put("reduceQueryFieldsPerShard", false);
         updatedValues.put("reduceQueryFieldsPerShard", true);
+        defaultValues.put("rebuildDatatypeFilter", false);
+        updatedValues.put("rebuildDatatypeFilter", true);
+        defaultValues.put("rebuildDatatypeFilterPerShard", false);
+        updatedValues.put("rebuildDatatypeFilterPerShard", true);
         defaultValues.put("reduceTypeMetadata", false);
         updatedValues.put("reduceTypeMetadata", true);
         defaultValues.put("reduceTypeMetadataPerShard", false);

diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
@@ -89,6 +89,8 @@
 
         <property name="auditType" value="ACTIVE" />
         <property name="logicDescription" value="Event query with rewritten logic" />
+        <property name="rebuildDatatypeFilter" value="true" />
+        <property name="rebuildDatatypeFilterPerShard" value="false" />
         <!-- Determines how many events in the global index lookup will be
         aggregated into a day range -->
         <property name="eventPerDayThreshold" value="40000" />

diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
@@ -251,6 +251,8 @@
         <property name="tfAggregationThresholdMs" value="-1" />
         <!--   enable per-tablet pruning of certain query options     -->
         <property name="pruneQueryOptions" value="false" />
+        <property name="rebuildDatatypeFilter" value="true" />
+        <property name="rebuildDatatypeFilterPerShard" value="false" />
         <property name="reduceIngestTypes" value="true"/>
         <property name="reduceIngestTypesPerShard" value="true"/>
         <property name="reduceQueryFields" value="false" />