diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml index 0d8bf59823acf..c18046f873477 100644 --- a/.idea/runConfigurations/Debug_OpenSearch.xml +++ b/.idea/runConfigurations/Debug_OpenSearch.xml @@ -6,6 +6,10 @@ <option name="HOST" value="localhost" /> <option name="PORT" value="5005" /> <option name="AUTO_RESTART" value="true" /> + <RunnerSettings RunnerId="Debug"> + <option name="DEBUG_PORT" value="5005" /> + <option name="LOCAL" value="false" /> + </RunnerSettings> <method v="2" /> </configuration> -</component> +</component> \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 7d791ace44682..4c53f3bc972f5 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -237,6 +237,7 @@ public IndexService( // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( + this.indexSettings.shouldWidenIndexSortType(), mapperService::fieldType, (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup) ); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index def7732e952ec..b80ce6f2694ca 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -619,6 +619,8 @@ public final class IndexSettings { private volatile long retentionLeaseMillis; + private final boolean widenIndexSortType; + /** * The maximum age of a retention lease before it is considered expired. * @@ -821,6 +823,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME); mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED); setMergeOnFlushPolicy(scopedSettings.get(INDEX_MERGE_ON_FLUSH_POLICY)); + System.out.println("CHETAN LOGSSSS " +IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)); + widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrBefore(LegacyESVersion.V_2_7_1); + System.out.println("CHETAN LOGSSSS " +widenIndexSortType); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer( @@ -1565,4 +1570,8 @@ private void setMergeOnFlushPolicy(String policy) { public Optional<UnaryOperator<MergePolicy>> getMergeOnFlushPolicy() { return Optional.ofNullable(mergeOnFlushPolicy); } + + public boolean shouldWidenIndexSortType() { + return this.widenIndexSortType; + } } diff --git a/server/src/main/java/org/opensearch/index/IndexSortConfig.java b/server/src/main/java/org/opensearch/index/IndexSortConfig.java index f73f96df4f9ad..15416bfd11069 100644 --- a/server/src/main/java/org/opensearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/opensearch/index/IndexSortConfig.java @@ -200,6 +200,7 @@ public boolean hasPrimarySortOnField(String field) { * or returns null if this index has no sort. */ public Sort buildIndexSort( + boolean shouldWidenIndexSortTpe, Function<String, MappedFieldType> fieldTypeLookup, BiFunction<MappedFieldType, Supplier<SearchLookup>, IndexFieldData<?>> fieldDataLookup ) { @@ -230,7 +231,11 @@ public Sort buildIndexSort( if (fieldData == null) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); } - sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse); + if(shouldWidenIndexSortTpe == true) { + sortFields[i] = fieldData.indexSortField(sortSpec.missingValue, mode, null, reverse); + } else { + sortFields[i] = fieldData.sortField(sortSpec.missingValue, mode, null, reverse); + } validateIndexSortField(sortFields[i]); } return new Sort(sortFields); diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 14f317ecaabca..ca1e77532197b 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -34,6 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DirectoryReader; @@ -2431,6 +2433,31 @@ private IndexWriter createWriter() throws IOException { // pkg-private for testing IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException { + /*List<Segment> segmetns = segments(false); + System.out.println("Getting info"); + if(segmetns != null && segmetns.size() > 0) { + System.out.println("Getting info size > 0"); + Sort indexSortFromSegment = segmetns.get(0).segmentSort; + if(indexSortFromSegment != null && indexSortFromSegment.getSort() != null) { + System.out.println("Indexsort from segment not null"); + SortField[] sortFieldsEngineConfig = iwc.getIndexSort().getSort(); + SortField[] sortFieldsSegmentInfo = indexSortFromSegment.getSort(); + for(int i = 0; i < sortFieldsSegmentInfo.length; i++) { + System.out.println("Comapring for : " + i); + if(i < sortFieldsEngineConfig.length && sortFieldsEngineConfig[i].getType() != sortFieldsSegmentInfo[i].getType()) { + System.out.println("Comparison failed : " + i); + if(sortFieldsSegmentInfo[i].getType() == SortField.Type.LONG) { + System.out.println("Comparison failed LONG deteted: " + i); + if(sortFieldsEngineConfig[i].getType() == SortField.Type.INT) { + System.out.println("Comparison failed INT detected : " + i); + sortFieldsEngineConfig[i] = sortFieldsSegmentInfo[i]; + System.out.println("Comparison failed segment info changed : " + i); + } + } + } + } + } + }*/ if (Assertions.ENABLED) { return new AssertingIndexWriter(directory, iwc); } else { diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java index 25437fb001978..9ac895d722571 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java @@ -94,6 +94,13 @@ public interface IndexFieldData<FD extends LeafFieldData> { */ SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse); + /** + * Returns the {@link SortField} to use for index sorting. + */ + default SortField indexSortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { + return sortField(missingValue, sortMode, nested, reverse); + } + /** * Build a sort implementation specialized for aggregations. */ diff --git a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java index 151617224d026..f1a9aee8c4c8d 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/opensearch/index/fielddata/IndexNumericFieldData.java @@ -79,12 +79,15 @@ public enum NumericType { private final boolean floatingPoint; private final ValuesSourceType valuesSourceType; - private final SortField.Type sortFieldType; + private SortField.Type sortFieldType; + + private boolean usePointBasedOptimization; NumericType(boolean floatingPoint, SortField.Type sortFieldType, ValuesSourceType valuesSourceType) { this.floatingPoint = floatingPoint; this.sortFieldType = sortFieldType; this.valuesSourceType = valuesSourceType; + this.usePointBasedOptimization = true; } public final boolean isFloatingPoint() { @@ -94,6 +97,11 @@ public final boolean isFloatingPoint() { public final ValuesSourceType getValuesSourceType() { return valuesSourceType; } + + public void setSortFieldType(SortField.Type type) { + this.sortFieldType = type; + this.usePointBasedOptimization = false; // Disable optimization if we set this + } } /** @@ -126,6 +134,7 @@ public final SortField sortField( || nested != null || (sortMode != MultiValueMode.MAX && sortMode != MultiValueMode.MIN) || targetNumericType != getNumericType()) { + System.out.println("Custom comparator logic....."); return new SortField(getFieldName(), source, reverse); } @@ -134,6 +143,9 @@ public final SortField sortField( : SortedNumericSelector.Type.MIN; SortField sortField = new SortedNumericSortField(getFieldName(), getNumericType().sortFieldType, reverse, selectorType); sortField.setMissingValue(source.missingObject(missingValue, reverse)); + if(getNumericType().usePointBasedOptimization == false) { + sortField.setOptimizeSortWithPoints(false); + } return sortField; } @@ -149,6 +161,27 @@ public final SortField sortField(Object missingValue, MultiValueMode sortMode, N return sortField(getNumericType(), missingValue, sortMode, nested, reverse); } + @Override + public final SortField indexSortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { + switch(getNumericType().sortFieldType) { + case INT: + getNumericType().setSortFieldType(LONG.sortFieldType); + break; + } + return sortField(getNumericType(), missingValue, sortMode, nested, reverse); + } + + + + public void modifySortFieldTypeForIndexSortIfNeeded() { + switch(this.sortFieldType) { + case INT: + this.sortFieldType = LONG.sortFieldType; + this.isIndexSort = true; + break; + } + } + /** * Builds a {@linkplain BucketedSort} for the {@code targetNumericType}, * casting the values if their native type doesn't match. @@ -207,8 +240,12 @@ private XFieldComparatorSource comparatorSource( case DATE_NANOSECONDS: return dateNanosComparatorSource(missingValue, sortMode, nested); case LONG: + System.out.println("CHETAN logs LONG"); return new LongValuesComparatorSource(this, missingValue, sortMode, nested); default: + if(getNumericType().sortFieldType == SortField.Type.LONG) { + return new LongValuesComparatorSource(this, missingValue, sortMode, nested); + } assert !targetNumericType.isFloatingPoint(); return new IntValuesComparatorSource(this, missingValue, sortMode, nested); } diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 05104b6cae7d7..324209bff1a50 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -429,6 +429,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { } IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData; NumericType resolvedType = resolveNumericType(numericType); + System.out.println("CHETAN LOGS : " + resolvedType); field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse); isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS; } else {