AOS 2.7 upgrade issue 2

gashutos · Sep 14, 2023 · b4a09da · b4a09da
1 parent eea21c3
commit b4a09da
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 3 deletions.
diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -237,6 +237,7 @@ public IndexService(
                 // The sort order is validated right after the merge of the mapping later in the process.
                 this.indexSortSupplier = () -> indexSettings.getIndexSortConfig()
                     .buildIndexSort(
+                        this.indexSettings.shouldWidenIndexSortType(),
                         mapperService::fieldType,
                         (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup)
                     );

diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -619,6 +619,8 @@ public final class IndexSettings {
 
     private volatile long retentionLeaseMillis;
 
+    private final boolean widenIndexSortType;
+
     /**
      * The maximum age of a retention lease before it is considered expired.
      *
@@ -821,6 +823,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME);
         mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED);
         setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY));
+        widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrBefore(LegacyESVersion.V_2_6_1);
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(
@@ -1565,4 +1568,8 @@ private void setMergeOnFlushPolicy(String policy) {
     public Optional<UnaryOperator<MergePolicy>> getMergeOnFlushPolicy() {
         return Optional.ofNullable(mergeOnFlushPolicy);
     }
+
+    public boolean shouldWidenIndexSortType() {
+        return this.widenIndexSortType;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
@@ -200,6 +200,7 @@ public boolean hasPrimarySortOnField(String field) {
      * or returns null if this index has no sort.
      */
     public Sort buildIndexSort(
+        boolean shouldWidenIndexSortTpe,
         Function<String, MappedFieldType> fieldTypeLookup,
         BiFunction<MappedFieldType, Supplier<SearchLookup>, IndexFieldData<?>> fieldDataLookup
     ) {
@@ -230,7 +231,11 @@ public Sort buildIndexSort(
             if (fieldData == null) {
                 throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]");
             }
-            sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            if(shouldWidenIndexSortTpe == true) {
+                sortFields[i] = fieldData.indexSortField(sortSpec.missingValue, mode, null, reverse);
+            } else {
+                sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            }
             validateIndexSortField(sortFields[i]);
         }
         return new Sort(sortFields);

diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -34,6 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;

diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
@@ -94,6 +94,13 @@ public interface IndexFieldData<FD extends LeafFieldData> {
      */
     SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse);
 
+    /**
+     * Returns the {@link SortField} to use for index sorting.
+     */
+    default SortField indexSortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        return sortField(missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Build a sort implementation specialized for aggregations.
      */

diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
@@ -79,12 +79,15 @@ public enum NumericType {
 
         private final boolean floatingPoint;
         private final ValuesSourceType valuesSourceType;
-        private final SortField.Type sortFieldType;
+        private SortField.Type sortFieldType;
+
+        private boolean usePointBasedOptimization;
 
         NumericType(boolean floatingPoint, SortField.Type sortFieldType, ValuesSourceType valuesSourceType) {
             this.floatingPoint = floatingPoint;
             this.sortFieldType = sortFieldType;
             this.valuesSourceType = valuesSourceType;
+            this.usePointBasedOptimization = true;
         }
 
         public final boolean isFloatingPoint() {
@@ -94,6 +97,11 @@ public final boolean isFloatingPoint() {
         public final ValuesSourceType getValuesSourceType() {
             return valuesSourceType;
         }
+
+        public void setSortFieldType(SortField.Type type) {
+            this.sortFieldType = type;
+            this.usePointBasedOptimization = false; // Disable optimization if we set this
+        }
     }
 
     /**
@@ -126,6 +134,7 @@ public final SortField sortField(
             || nested != null
             || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
             || targetNumericType != getNumericType()) {
+            System.out.println("Custom comparator logic.....");
             return new SortField(getFieldName(), source, reverse);
         }
 
@@ -134,6 +143,9 @@ public final SortField sortField(
             : SortedNumericSelector.Type.MIN;
         SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType);
         sortField.setMissingValue(source.missingObject(missingValue, reverse));
+        if(getNumericType().usePointBasedOptimization == false) {
+            sortField.setOptimizeSortWithPoints(false);
+        }
         return sortField;
     }
 
@@ -149,6 +161,16 @@ public final SortField sortField(Object missingValue, MultiValueMode sortMode, N
         return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
     }
 
+    @Override
+    public final SortField indexSortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        switch(getNumericType().sortFieldType) {
+            case INT:
+                getNumericType().setSortFieldType(NumericType.LONG.sortFieldType);
+                break;
+        }
+        return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Builds a {@linkplain BucketedSort} for the {@code targetNumericType},
      * casting the values if their native type doesn't match.
@@ -209,6 +231,9 @@ private XFieldComparatorSource comparatorSource(
             case LONG:
                 return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
             default:
+                if(getNumericType().sortFieldType == SortField.Type.LONG) {
+                    return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
+                }
                 assert !targetNumericType.isFloatingPoint();
                 return new IntValuesComparatorSource(this, missingValue, sortMode, nested);
         }

diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
@@ -429,6 +429,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
             }
             IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData;
             NumericType resolvedType = resolveNumericType(numericType);
+            System.out.println("CHETAN LOGS : " + resolvedType);
             field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
             isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS;
         } else {