Merge remote-tracking branch 'origin/integration' into integration

NationalSecurityAgency · Sep 18, 2024 · 8db4eae · 8db4eae
2 parents a7d6787 + dced884
commit 8db4eae
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 9 deletions.
diff --git a/core/query/src/main/java/datawave/core/query/logic/composite/CompositeQueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/composite/CompositeQueryLogic.java
@@ -250,8 +250,6 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting
         StringBuilder logicQueryStringBuilder = new StringBuilder();
         if (!getInitializedLogics().isEmpty()) {
             logicQueryStringBuilder.append(getConfig().getQueryString());
-        } else {
-            logicQueryStringBuilder.append("CompositeQueryLogic: ");
         }
 
         Map<String,Exception> exceptions = new HashMap<>();
@@ -266,7 +264,9 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting
                 if (logicQueryStringBuilder.length() > 0) {
                     logicQueryStringBuilder.append(" || ");
                 }
-                logicQueryStringBuilder.append("( ( logic = '").append(logicName).append("' )").append(" && ");
+
+                logicQueryStringBuilder.append("( ");
+                logicQueryStringBuilder.append("( logic = '").append(logicName).append("' )");
 
                 try {
                     // duplicate the settings for this query
@@ -277,12 +277,14 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting
 
                     config = logic.initialize(client, settingsCopy, runtimeQueryAuthorizations);
 
+                    logicQueryStringBuilder.append(" && ").append("( queryId = '").append(settingsCopy.getId()).append("' )");
+
                     // only add this query logic to the initialized logic states if it was not simply filtered out
                     if (logic instanceof FilteredQueryLogic && ((FilteredQueryLogic) logic).isFiltered()) {
                         log.info("Dropping " + logic.getLogicName() + " as it was filtered out");
-                        logicQueryStringBuilder.append("( filtered = true )");
+                        logicQueryStringBuilder.append(" && ").append("( filtered = true )");
                     } else {
-                        logicQueryStringBuilder.append(config.getQueryString());
+                        logicQueryStringBuilder.append(" && ").append(config.getQueryString());
                         QueryLogicHolder holder = new QueryLogicHolder(logicName, logic);
                         holder.setSettings(settingsCopy);
                         holder.setMaxResults(logic.getResultLimit(settingsCopy));
@@ -298,7 +300,7 @@ public GenericQueryConfiguration initialize(AccumuloClient client, Query setting
                 } catch (Exception e) {
                     exceptions.put(logicName, e);
                     log.error("Failed to initialize " + logic.getClass().getName(), e);
-                    logicQueryStringBuilder.append("( failure = '").append(e.getMessage()).append("' )");
+                    logicQueryStringBuilder.append(" && ").append("( failure = '").append(e.getMessage()).append("' )");
                     failedQueryLogics.put(logicName, logic);
                 } finally {
                     queryLogics.remove(next.getKey());

diff --git a/...ery-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java b/...ery-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java
@@ -769,8 +769,8 @@ protected void findTop() throws IOException {
                     // no need to check containership if not returning sorted uids
                     if (!sortedUIDs || this.lastRangeSeeked.contains(key)) {
                         this.topKey = key;
-                        if (log.isDebugEnabled()) {
-                            log.debug("setting as topKey " + topKey);
+                        if (log.isTraceEnabled()) {
+                            log.trace("setting as topKey " + topKey);
                         }
                         break;
                     }
@@ -879,6 +879,7 @@ private void fillSortedSets() throws IOException {
         if (log.isDebugEnabled()) {
             log.debug("Processing " + boundingFiRanges + " for " + this);
         }
+        long startFillSets = System.currentTimeMillis();
 
         TotalResults totalResults = new TotalResults(maxResults);
 
@@ -916,8 +917,11 @@ private void fillSortedSets() throws IOException {
             }
         }
 
+        long fillSetTiming = System.currentTimeMillis() - startFillSets;
+        log.info("Filled ivarator sets for " + boundingFiRanges.size() + " ranges took " + fillSetTiming + "ms for " + this);
+
         if (failed) {
-            log.error("Failed to complete ivarator cache: " + result, exception);
+            log.error("Failed to complete ivarator cache: " + result + " for " + this, exception);
             throw new IvaratorException("Failed to complete ivarator cache: " + result, exception);
         }
 
@@ -1102,6 +1106,7 @@ protected Future<?> fillSet(final Range boundingFiRange, final TotalResults tota
 
         // create runnable
         Runnable runnable = () -> {
+            long startFillSet = System.currentTimeMillis();
             if (log.isDebugEnabled()) {
                 log.debug("Starting fillSet(" + boundingFiRange + ')');
             }
@@ -1210,6 +1215,8 @@ protected Future<?> fillSet(final Range boundingFiRange, final TotalResults tota
                 log.error("Failed to complete fillSet(" + boundingFiRange + ")", e);
                 throw new RuntimeException(e);
             } finally {
+                long timing = System.currentTimeMillis() - startFillSet;
+                log.info("Completed " + boundingFiRange + " ivarator in " + timing + "ms");
                 // return the ivarator source back to the pool.
                 returnPoolSource(source);
                 if (log.isDebugEnabled()) {
@@ -1644,4 +1651,13 @@ public void setCollectTimingDetails(boolean collectTimingDetails) {
     public void setQuerySpanCollector(QuerySpanCollector querySpanCollector) {
         this.querySpanCollector = querySpanCollector;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+        builder.append("DatawaveFieldIndexCachingIteratorJexl (").append(queryId).append(") fName=").append(getFieldName()).append(", fValue=")
+                        .append(getFieldValue()).append(", negated=").append(isNegated()).append("}");
+        return builder.toString();
+    }
+
 }
diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/FederatedQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/FederatedQueryPlanner.java
@@ -13,6 +13,7 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.UUID;
 import java.util.stream.Collectors;
 
 import org.apache.accumulo.core.client.TableNotFoundException;
@@ -320,6 +321,7 @@ public CloseableIterable<QueryData> process(GenericQueryConfiguration genericCon
         FederatedQueryIterable results = new FederatedQueryIterable();
         int totalProcessed = 1;
         ShardQueryConfiguration firstConfigCopy = null;
+        UUID queryId = originalConfig.getQuery().getId();
         for (Pair<Date,Date> dateRange : dateRanges) {
             // Format the start and end date of the current sub-query to execute.
             String subStartDate = dateFormat.format(dateRange.getLeft());
@@ -334,10 +336,14 @@ public CloseableIterable<QueryData> process(GenericQueryConfiguration genericCon
             configCopy.setBeginDate(dateRange.getLeft());
             configCopy.setEndDate(dateRange.getRight());
 
+            // we want to make sure the same query id for tracking purposes and execution
+            configCopy.getQuery().setId(queryId);
+
             // Create a copy of the original default query planner, and process the query with the new date range.
             DefaultQueryPlanner subPlan = this.queryPlanner.clone();
 
             try {
+
                 CloseableIterable<QueryData> queryData = subPlan.process(configCopy, query, settings, scannerFactory);
                 results.addIterable(queryData);
             } catch (Exception e) {