AOS 2.7 upgrade issue 2

gashutos · Sep 14, 2023 · 18c4811 · 18c4811
1 parent 28724ce
commit 18c4811
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 3 deletions.
diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -242,6 +242,7 @@ public IndexService(
                 // The sort order is validated right after the merge of the mapping later in the process.
                 this.indexSortSupplier = () -> indexSettings.getIndexSortConfig()
                     .buildIndexSort(
+                        this.indexSettings.shouldWidenIndexSortType(),
                         mapperService::fieldType,
                         (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup)
                     );

diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -660,6 +660,7 @@ public final class IndexSettings {
     private volatile long retentionLeaseMillis;
 
     private volatile String defaultSearchPipeline;
+    private final boolean widenIndexSortType;
 
     /**
      * The maximum age of a retention lease before it is considered expired.
@@ -857,6 +858,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED);
         setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY));
         defaultSearchPipeline = scopedSettings.get(DEFAULT_SEARCH_PIPELINE);
+        widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrBefore(LegacyESVersion.V_2_6_1);
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(
@@ -1652,4 +1654,8 @@ public String getDefaultSearchPipeline() {
     public void setDefaultSearchPipeline(String defaultSearchPipeline) {
         this.defaultSearchPipeline = defaultSearchPipeline;
     }
+
+    public boolean shouldWidenIndexSortType() {
+        return this.widenIndexSortType;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
@@ -200,6 +200,7 @@ public boolean hasPrimarySortOnField(String field) {
      * or returns null if this index has no sort.
      */
     public Sort buildIndexSort(
+        boolean shouldWidenIndexSortTpe,
         Function<String, MappedFieldType> fieldTypeLookup,
         BiFunction<MappedFieldType, Supplier<SearchLookup>, IndexFieldData<?>> fieldDataLookup
     ) {
@@ -230,7 +231,11 @@ public Sort buildIndexSort(
             if (fieldData == null) {
                 throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]");
             }
-            sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            if(shouldWidenIndexSortTpe == true) {
+                sortFields[i] = fieldData.indexSortField(sortSpec.missingValue, mode, null, reverse);
+            } else {
+                sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            }
             validateIndexSortField(sortFields[i]);
         }
         return new Sort(sortFields);

diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -33,6 +33,8 @@
 package org.opensearch.index.engine;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;

diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
@@ -94,6 +94,13 @@ public interface IndexFieldData<FD extends LeafFieldData> {
      */
     SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse);
 
+    /**
+     * Returns the {@link SortField} to use for index sorting.
+     */
+    default SortField indexSortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        return sortField(missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Build a sort implementation specialized for aggregations.
      */

diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
@@ -81,12 +81,15 @@ public enum NumericType {
 
         private final boolean floatingPoint;
         private final ValuesSourceType valuesSourceType;
-        private final SortField.Type sortFieldType;
+        private SortField.Type sortFieldType;
+
+        private boolean usePointBasedOptimization;
 
         NumericType(boolean floatingPoint, SortField.Type sortFieldType, ValuesSourceType valuesSourceType) {
             this.floatingPoint = floatingPoint;
             this.sortFieldType = sortFieldType;
             this.valuesSourceType = valuesSourceType;
+            this.usePointBasedOptimization = true;
         }
 
         public final boolean isFloatingPoint() {
@@ -96,6 +99,11 @@ public final boolean isFloatingPoint() {
         public final ValuesSourceType getValuesSourceType() {
             return valuesSourceType;
         }
+
+        public void setSortFieldType(SortField.Type type) {
+            this.sortFieldType = type;
+            this.usePointBasedOptimization = false; // Disable optimization if we set this
+        }
     }
 
     /**
@@ -128,6 +136,7 @@ public final SortField sortField(
             || nested != null
             || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
             || targetNumericType != getNumericType()) {
+            System.out.println("Custom comparator logic.....");
             return new SortField(getFieldName(), source, reverse);
         }
 
@@ -136,6 +145,9 @@ public final SortField sortField(
             : SortedNumericSelector.Type.MIN;
         SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType);
         sortField.setMissingValue(source.missingObject(missingValue, reverse));
+        if(getNumericType().usePointBasedOptimization == false) {
+            sortField.setOptimizeSortWithPoints(false);
+        }
         return sortField;
     }
 
@@ -151,6 +163,16 @@ public final SortField sortField(Object missingValue, MultiValueMode sortMode, N
         return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
     }
 
+    @Override
+    public final SortField indexSortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        switch(getNumericType().sortFieldType) {
+            case INT:
+                getNumericType().setSortFieldType(NumericType.LONG.sortFieldType);
+                break;
+        }
+        return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Builds a {@linkplain BucketedSort} for the {@code targetNumericType},
      * casting the values if their native type doesn't match.
@@ -220,6 +242,9 @@ private XFieldComparatorSource comparatorSource(
                 source = new LongValuesComparatorSource(this, missingValue, sortMode, nested);
                 break;
             default:
+                if(getNumericType().sortFieldType == SortField.Type.LONG) {
+                    return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
+                }
                 assert !targetNumericType.isFloatingPoint();
                 source = new IntValuesComparatorSource(this, missingValue, sortMode, nested);
         }

diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
@@ -428,6 +428,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
             }
             IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData;
             NumericType resolvedType = resolveNumericType(numericType);
+            System.out.println("CHETAN LOGS : " + resolvedType);
             field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
             isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS;
         } else {