From 7210f3782ec59f77f910eccfa19f14342d48adad Mon Sep 17 00:00:00 2001
From: gashutos <gashutos@amazon.com>
Date: Thu, 14 Sep 2023 14:59:00 +0530
Subject: [PATCH] AOS 2.7 upgrade issue

---
 .idea/runConfigurations/Debug_OpenSearch.xml  |  6 +++-
 .../org/opensearch/index/IndexService.java    |  1 +
 .../org/opensearch/index/IndexSettings.java   |  9 ++++++
 .../org/opensearch/index/IndexSortConfig.java |  7 ++++-
 .../index/engine/InternalEngine.java          | 27 ++++++++++++++++++
 .../index/fielddata/IndexFieldData.java       |  7 +++++
 .../fielddata/IndexNumericFieldData.java      | 28 ++++++++++++++++++-
 .../search/sort/FieldSortBuilder.java         |  1 +
 8 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..c18046f873477 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,6 +6,10 @@
     <option name="HOST" value="localhost" />
     <option name="PORT" value="5005" />
     <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
     <method v="2" />
   </configuration>
-</component>
+</component>
\ No newline at end of file
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index 7d791ace44682..4c53f3bc972f5 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -237,6 +237,7 @@ public IndexService(
                 // The sort order is validated right after the merge of the mapping later in the process.
                 this.indexSortSupplier = () -> indexSettings.getIndexSortConfig()
                     .buildIndexSort(
+                        this.indexSettings.shouldWidenIndexSortType(),
                         mapperService::fieldType,
                         (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup)
                     );
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index def7732e952ec..b80ce6f2694ca 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -619,6 +619,8 @@ public final class IndexSettings {
 
     private volatile long retentionLeaseMillis;
 
+    private final boolean widenIndexSortType;
+
     /**
      * The maximum age of a retention lease before it is considered expired.
      *
@@ -821,6 +823,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME);
         mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED);
         setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY));
+        System.out.println("CHETAN LOGSSSS    "  +IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings));
+        widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrBefore(LegacyESVersion.V_2_7_1);
+        System.out.println("CHETAN LOGSSSS    "  +widenIndexSortType);
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(
@@ -1565,4 +1570,8 @@ private void setMergeOnFlushPolicy(String policy) {
     public Optional<UnaryOperator<MergePolicy>> getMergeOnFlushPolicy() {
         return Optional.ofNullable(mergeOnFlushPolicy);
     }
+
+    public boolean shouldWidenIndexSortType() {
+        return this.widenIndexSortType;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
index f73f96df4f9ad..15416bfd11069 100644
--- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java
+++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java
@@ -200,6 +200,7 @@ public boolean hasPrimarySortOnField(String field) {
      * or returns null if this index has no sort.
      */
     public Sort buildIndexSort(
+        boolean shouldWidenIndexSortTpe,
         Function<String, MappedFieldType> fieldTypeLookup,
         BiFunction<MappedFieldType, Supplier<SearchLookup>, IndexFieldData<?>> fieldDataLookup
     ) {
@@ -230,7 +231,11 @@ public Sort buildIndexSort(
             if (fieldData == null) {
                 throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]");
             }
-            sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            if(shouldWidenIndexSortTpe == true) {
+                sortFields[i] = fieldData.indexSortField(sortSpec.missingValue, mode, null, reverse);
+            } else {
+                sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse);
+            }
             validateIndexSortField(sortFields[i]);
         }
         return new Sort(sortFields);
diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
index 14f317ecaabca..ca1e77532197b 100644
--- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -34,6 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
@@ -2431,6 +2433,31 @@ private IndexWriter createWriter() throws IOException {
 
     // pkg-private for testing
     IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
+        /*List<Segment> segmetns = segments(false);
+        System.out.println("Getting info");
+        if(segmetns != null && segmetns.size() > 0) {
+            System.out.println("Getting info size > 0");
+            Sort indexSortFromSegment = segmetns.get(0).segmentSort;
+            if(indexSortFromSegment != null && indexSortFromSegment.getSort() != null) {
+                System.out.println("Indexsort from segment not null");
+                SortField[] sortFieldsEngineConfig = iwc.getIndexSort().getSort();
+                SortField[] sortFieldsSegmentInfo = indexSortFromSegment.getSort();
+                for(int i = 0; i < sortFieldsSegmentInfo.length; i++) {
+                    System.out.println("Comapring for : " + i);
+                    if(i < sortFieldsEngineConfig.length && sortFieldsEngineConfig[i].getType() != sortFieldsSegmentInfo[i].getType()) {
+                        System.out.println("Comparison failed : " + i);
+                        if(sortFieldsSegmentInfo[i].getType() == SortField.Type.LONG) {
+                            System.out.println("Comparison failed LONG deteted: " + i);
+                            if(sortFieldsEngineConfig[i].getType() == SortField.Type.INT) {
+                                System.out.println("Comparison failed INT detected : " + i);
+                                sortFieldsEngineConfig[i] = sortFieldsSegmentInfo[i];
+                                System.out.println("Comparison failed segment info changed : " + i);
+                            }
+                        }
+                    }
+                }
+            }
+        }*/
         if (Assertions.ENABLED) {
             return new AssertingIndexWriter(directory, iwc);
         } else {
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
index 25437fb001978..9ac895d722571 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java
@@ -94,6 +94,13 @@ public interface IndexFieldData<FD extends LeafFieldData> {
      */
     SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse);
 
+    /**
+     * Returns the {@link SortField} to use for index sorting.
+     */
+    default SortField indexSortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        return sortField(missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Build a sort implementation specialized for aggregations.
      */
diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
index 151617224d026..30834f3978f06 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java
@@ -79,12 +79,15 @@ public enum NumericType {
 
         private final boolean floatingPoint;
         private final ValuesSourceType valuesSourceType;
-        private final SortField.Type sortFieldType;
+        private SortField.Type sortFieldType;
+
+        private boolean usePointBasedOptimization;
 
         NumericType(boolean floatingPoint, SortField.Type sortFieldType, ValuesSourceType valuesSourceType) {
             this.floatingPoint = floatingPoint;
             this.sortFieldType = sortFieldType;
             this.valuesSourceType = valuesSourceType;
+            this.usePointBasedOptimization = true;
         }
 
         public final boolean isFloatingPoint() {
@@ -94,6 +97,11 @@ public final boolean isFloatingPoint() {
         public final ValuesSourceType getValuesSourceType() {
             return valuesSourceType;
         }
+
+        public void setSortFieldType(SortField.Type type) {
+            this.sortFieldType = type;
+            this.usePointBasedOptimization = false; // Disable optimization if we set this
+        }
     }
 
     /**
@@ -126,6 +134,7 @@ public final SortField sortField(
             || nested != null
             || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN)
             || targetNumericType != getNumericType()) {
+            System.out.println("Custom comparator logic.....");
             return new SortField(getFieldName(), source, reverse);
         }
 
@@ -134,6 +143,9 @@ public final SortField sortField(
             : SortedNumericSelector.Type.MIN;
         SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType);
         sortField.setMissingValue(source.missingObject(missingValue, reverse));
+        if(getNumericType().usePointBasedOptimization == false) {
+            sortField.setOptimizeSortWithPoints(false);
+        }
         return sortField;
     }
 
@@ -149,6 +161,16 @@ public final SortField sortField(Object missingValue, MultiValueMode sortMode, N
         return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
     }
 
+    @Override
+    public final SortField indexSortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        switch(getNumericType().sortFieldType) {
+            case INT:
+                getNumericType().setSortFieldType(NumericType.LONG.sortFieldType);
+                break;
+        }
+        return sortField(getNumericType(), missingValue, sortMode, nested, reverse);
+    }
+
     /**
      * Builds a {@linkplain BucketedSort} for the {@code targetNumericType},
      * casting the values if their native type doesn't match.
@@ -207,8 +229,12 @@ private XFieldComparatorSource comparatorSource(
             case DATE_NANOSECONDS:
                 return dateNanosComparatorSource(missingValue, sortMode, nested);
             case LONG:
+                System.out.println("CHETAN logs LONG");
                 return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
             default:
+                if(getNumericType().sortFieldType == SortField.Type.LONG) {
+                    return new LongValuesComparatorSource(this, missingValue, sortMode, nested);
+                }
                 assert !targetNumericType.isFloatingPoint();
                 return new IntValuesComparatorSource(this, missingValue, sortMode, nested);
         }
diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
index 05104b6cae7d7..324209bff1a50 100644
--- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
+++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java
@@ -429,6 +429,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
             }
             IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData;
             NumericType resolvedType = resolveNumericType(numericType);
+            System.out.println("CHETAN LOGS : " + resolvedType);
             field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
             isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS;
         } else {