null.
- */
- private static UnsignedLongMultiValueMode fromString(String sortMode) {
- try {
- return valueOf(sortMode.toUpperCase(Locale.ROOT));
- } catch (Exception e) {
- throw new IllegalArgumentException("Illegal sort mode: " + sortMode);
- }
- }
-
- /**
- * Convert a {@link MultiValueMode} to a {@link UnsignedLongMultiValueMode}.
- */
- public static UnsignedLongMultiValueMode toUnsignedSortMode(MultiValueMode sortMode) {
- return fromString(sortMode.name());
- }
-
- /**
- * Return a {@link NumericDocValues} instance that can be used to sort documents
- * with this mode and the provided values. When a document has no value,
- * missingValue is returned.
- * - * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX - */ - public NumericDocValues select(final SortedNumericDocValues values) { - final NumericDocValues singleton = DocValues.unwrapSingleton(values); - if (singleton != null) { - return singleton; - } else { - return new AbstractNumericDocValues() { - - private long value; - - @Override - public boolean advanceExact(int target) throws IOException { - if (values.advanceExact(target)) { - value = pick(values); - return true; - } - return false; - } - - @Override - public int docID() { - return values.docID(); - } - - @Override - public long longValue() throws IOException { - return value; - } - }; - } - } - - protected long pick(SortedNumericDocValues values) throws IOException { - throw new IllegalArgumentException("Unsupported sort mode: " + this); - } - - /** - * Return a {@link NumericDocValues} instance that can be used to sort root documents - * with this mode, the provided values and filters for root/inner documents. - *
- * For every root document, the values of its inner documents will be aggregated.
- * If none of the inner documents has a value, then missingValue is returned.
- *
- * Allowed Modes: SUM, AVG, MIN, MAX - *
- * NOTE: Calling the returned instance on docs that are not root docs is illegal
- * The returned instance can only be evaluate the current and upcoming docs
- */
- public NumericDocValues select(
- final SortedNumericDocValues values,
- final long missingValue,
- final BitSet parentDocs,
- final DocIdSetIterator childDocs,
- int maxDoc,
- int maxChildren
- ) throws IOException {
- if (parentDocs == null || childDocs == null) {
- return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue);
- }
-
- return new AbstractNumericDocValues() {
-
- int lastSeenParentDoc = -1;
- long lastEmittedValue = missingValue;
-
- @Override
- public boolean advanceExact(int parentDoc) throws IOException {
- assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
- if (parentDoc == lastSeenParentDoc) {
- return true;
- } else if (parentDoc == 0) {
- lastEmittedValue = missingValue;
- return true;
- }
- final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
- final int firstChildDoc;
- if (childDocs.docID() > prevParentDoc) {
- firstChildDoc = childDocs.docID();
- } else {
- firstChildDoc = childDocs.advance(prevParentDoc + 1);
- }
-
- lastSeenParentDoc = parentDoc;
- lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren);
- return true;
- }
-
- @Override
- public int docID() {
- return lastSeenParentDoc;
- }
-
- @Override
- public long longValue() {
- return lastEmittedValue;
- }
- };
- }
-
- protected long pick(
- SortedNumericDocValues values,
- long missingValue,
- DocIdSetIterator docItr,
- int startDoc,
- int endDoc,
- int maxChildren
- ) throws IOException {
- throw new IllegalArgumentException("Unsupported sort mode: " + this);
- }
-
- /**
- * Copied from {@link Long#divideUnsigned(long, long)} and {@link Long#remainderUnsigned(long, long)}
- */
- private static long divideUnsignedAndRoundUp(long dividend, long divisor) {
- assert divisor > 0;
- final long q = (dividend >>> 1) / divisor << 1;
- final long r = dividend - q * divisor;
- final long quotient = q + ((r | ~(r - divisor)) >>> (Long.SIZE - 1));
- final long rem = r - ((~(r - divisor) >> (Long.SIZE - 1)) & divisor);
- return quotient + Math.round((double) rem / divisor);
- }
-}
diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java
index c7b9c9f445484..c9076e773ba29 100644
--- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java
+++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongValuesComparatorSource.java
@@ -24,6 +24,8 @@
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.IndexNumericFieldData;
import org.opensearch.index.fielddata.LeafNumericFieldData;
+import org.opensearch.index.fielddata.SingletonSortedNumericUnsignedLongValues;
+import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues;
import org.opensearch.index.search.comparators.UnsignedLongComparator;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
@@ -41,7 +43,6 @@
public class UnsignedLongValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData indexFieldData;
- private final UnsignedLongMultiValueMode unsignedSortMode;
public UnsignedLongValuesComparatorSource(
IndexNumericFieldData indexFieldData,
@@ -51,7 +52,6 @@ public UnsignedLongValuesComparatorSource(
) {
super(missingValue, sortMode, nested);
this.indexFieldData = indexFieldData;
- this.unsignedSortMode = UnsignedLongMultiValueMode.toUnsignedSortMode(sortMode);
}
@Override
@@ -59,21 +59,21 @@ public SortField.Type reducedType() {
return SortField.Type.LONG;
}
- private SortedNumericDocValues loadDocValues(LeafReaderContext context) {
+ private SortedNumericUnsignedLongValues loadDocValues(LeafReaderContext context) {
final LeafNumericFieldData data = indexFieldData.load(context);
SortedNumericDocValues values = data.getLongValues();
- return values;
+ return new SingletonSortedNumericUnsignedLongValues(values);
}
private NumericDocValues getNumericDocValues(LeafReaderContext context, BigInteger missingValue) throws IOException {
- final SortedNumericDocValues values = loadDocValues(context);
+ final SortedNumericUnsignedLongValues values = loadDocValues(context);
if (nested == null) {
- return FieldData.replaceMissing(unsignedSortMode.select(values), missingValue);
+ return FieldData.replaceMissing(sortMode.select(values), missingValue);
}
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
final int maxChildren = nested.getNestedSort() != null ? nested.getNestedSort().getMaxChildren() : Integer.MAX_VALUE;
- return unsignedSortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
+ return sortMode.select(values, missingValue.longValue(), rootDocs, innerDocs, context.reader().maxDoc(), maxChildren);
}
@Override
diff --git a/server/src/main/java/org/opensearch/search/MultiValueMode.java b/server/src/main/java/org/opensearch/search/MultiValueMode.java
index a99da674836f2..b9df357016788 100644
--- a/server/src/main/java/org/opensearch/search/MultiValueMode.java
+++ b/server/src/main/java/org/opensearch/search/MultiValueMode.java
@@ -42,6 +42,7 @@
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
+import org.opensearch.common.Numbers;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
@@ -53,6 +54,7 @@
import org.opensearch.index.fielddata.NumericDoubleValues;
import org.opensearch.index.fielddata.SortedBinaryDocValues;
import org.opensearch.index.fielddata.SortedNumericDoubleValues;
+import org.opensearch.index.fielddata.SortedNumericUnsignedLongValues;
import java.io.IOException;
import java.util.Locale;
@@ -143,6 +145,44 @@ protected double pick(
return totalCount > 0 ? totalValue : missingValue;
}
+
+ @Override
+ protected long pick(SortedNumericUnsignedLongValues values) throws IOException {
+ final int count = values.docValueCount();
+ long total = 0;
+ for (int index = 0; index < count; ++index) {
+ total += values.nextValue();
+ }
+ return total;
+ }
+
+ @Override
+ protected long pick(
+ SortedNumericUnsignedLongValues values,
+ long missingValue,
+ DocIdSetIterator docItr,
+ int startDoc,
+ int endDoc,
+ int maxChildren
+ ) throws IOException {
+ int totalCount = 0;
+ long totalValue = 0;
+ int count = 0;
+ for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
+ if (values.advanceExact(doc)) {
+ if (++count > maxChildren) {
+ break;
+ }
+
+ final int docCount = values.docValueCount();
+ for (int index = 0; index < docCount; ++index) {
+ totalValue += values.nextValue();
+ }
+ totalCount += docCount;
+ }
+ }
+ return totalCount > 0 ? totalValue : missingValue;
+ }
},
/**
@@ -228,6 +268,46 @@ protected double pick(
}
return totalValue / totalCount;
}
+
+ @Override
+ protected long pick(SortedNumericUnsignedLongValues values) throws IOException {
+ final int count = values.docValueCount();
+ long total = 0;
+ for (int index = 0; index < count; ++index) {
+ total += values.nextValue();
+ }
+ return count > 1 ? divideUnsignedAndRoundUp(total, count) : total;
+ }
+
+ @Override
+ protected long pick(
+ SortedNumericUnsignedLongValues values,
+ long missingValue,
+ DocIdSetIterator docItr,
+ int startDoc,
+ int endDoc,
+ int maxChildren
+ ) throws IOException {
+ int totalCount = 0;
+ long totalValue = 0;
+ int count = 0;
+ for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
+ if (values.advanceExact(doc)) {
+ if (++count > maxChildren) {
+ break;
+ }
+ final int docCount = values.docValueCount();
+ for (int index = 0; index < docCount; ++index) {
+ totalValue += values.nextValue();
+ }
+ totalCount += docCount;
+ }
+ }
+ if (totalCount < 1) {
+ return missingValue;
+ }
+ return totalCount > 1 ? divideUnsignedAndRoundUp(totalValue, totalCount) : totalValue;
+ }
},
/**
@@ -259,6 +339,45 @@ protected double pick(SortedNumericDoubleValues values) throws IOException {
return values.nextValue();
}
}
+
+ @Override
+ protected long pick(SortedNumericUnsignedLongValues values) throws IOException {
+ int count = values.docValueCount();
+ long firstValue = values.nextValue();
+ if (count == 1) {
+ return firstValue;
+ } else if (count == 2) {
+ long total = firstValue + values.nextValue();
+ return (total >>> 1) + (total & 1);
+ } else if (firstValue >= 0) {
+ for (int i = 1; i < (count - 1) / 2; ++i) {
+ values.nextValue();
+ }
+ if (count % 2 == 0) {
+ long total = values.nextValue() + values.nextValue();
+ return (total >>> 1) + (total & 1);
+ } else {
+ return values.nextValue();
+ }
+ }
+
+ final long[] docValues = new long[count];
+ docValues[0] = firstValue;
+ int firstPositiveIndex = 0;
+ for (int i = 1; i < count; ++i) {
+ docValues[i] = values.nextValue();
+ if (docValues[i] >= 0 && firstPositiveIndex == 0) {
+ firstPositiveIndex = i;
+ }
+ }
+ final int mid = ((count - 1) / 2 + firstPositiveIndex) % count;
+ if (count % 2 == 0) {
+ long total = docValues[mid] + docValues[(mid + 1) % count];
+ return (total >>> 1) + (total & 1);
+ } else {
+ return docValues[mid];
+ }
+ }
},
/**
@@ -382,6 +501,47 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc
return hasValue ? ord : -1;
}
+
+ @Override
+ protected long pick(SortedNumericUnsignedLongValues values) throws IOException {
+ final int count = values.docValueCount();
+ final long min = values.nextValue();
+ if (count == 1 || min > 0) {
+ return min;
+ }
+ for (int i = 1; i < count; ++i) {
+ long val = values.nextValue();
+ if (val >= 0) {
+ return val;
+ }
+ }
+ return min;
+ }
+
+ @Override
+ protected long pick(
+ SortedNumericUnsignedLongValues values,
+ long missingValue,
+ DocIdSetIterator docItr,
+ int startDoc,
+ int endDoc,
+ int maxChildren
+ ) throws IOException {
+ boolean hasValue = false;
+ long minValue = Numbers.MAX_UNSIGNED_LONG_VALUE_AS_LONG;
+ int count = 0;
+ for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
+ if (values.advanceExact(doc)) {
+ if (++count > maxChildren) {
+ break;
+ }
+ final long docMin = pick(values);
+ minValue = Long.compareUnsigned(docMin, minValue) < 0 ? docMin : minValue;
+ hasValue = true;
+ }
+ }
+ return hasValue ? minValue : missingValue;
+ }
},
/**
@@ -525,6 +685,46 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc
}
return ord;
}
+
+ @Override
+ protected long pick(SortedNumericUnsignedLongValues values) throws IOException {
+ final int count = values.docValueCount();
+ long max = values.nextValue();
+ long val;
+ for (int i = 1; i < count; ++i) {
+ val = values.nextValue();
+ if (max < 0 && val >= 0) {
+ return max;
+ }
+ max = val;
+ }
+ return max;
+ }
+
+ @Override
+ protected long pick(
+ SortedNumericUnsignedLongValues values,
+ long missingValue,
+ DocIdSetIterator docItr,
+ int startDoc,
+ int endDoc,
+ int maxChildren
+ ) throws IOException {
+ boolean hasValue = false;
+ long maxValue = Numbers.MIN_UNSIGNED_LONG_VALUE_AS_LONG;
+ int count = 0;
+ for (int doc = startDoc; doc < endDoc; doc = docItr.nextDoc()) {
+ if (values.advanceExact(doc)) {
+ if (++count > maxChildren) {
+ break;
+ }
+ final long docMax = pick(values);
+ maxValue = Long.compareUnsigned(maxValue, docMax) < 0 ? docMax : maxValue;
+ hasValue = true;
+ }
+ }
+ return hasValue ? maxValue : missingValue;
+ }
};
/**
@@ -1032,6 +1232,122 @@ protected int pick(SortedDocValues values, DocIdSetIterator docItr, int startDoc
throw new IllegalArgumentException("Unsupported sort mode: " + this);
}
+ /**
+ * Return a {@link NumericDoubleValues} instance that can be used to sort documents
+ * with this mode and the provided values. When a document has no value,
+ * missingValue is returned.
+ *
+ * Allowed Modes: SUM, AVG, MEDIAN, MIN, MAX + */ + public NumericDocValues select(final SortedNumericUnsignedLongValues values) { + final SortedNumericDocValues sortedNumericDocValues = FieldData.unwrapSingleton(values); + final NumericDocValues singleton = DocValues.unwrapSingleton(sortedNumericDocValues); + if (singleton != null) { + return singleton; + } else { + return new AbstractNumericDocValues() { + + private long value; + + @Override + public boolean advanceExact(int target) throws IOException { + if (values.advanceExact(target)) { + value = pick(values); + return true; + } + return false; + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public long longValue() throws IOException { + return value; + } + }; + } + } + + protected long pick(SortedNumericUnsignedLongValues values) throws IOException { + throw new IllegalArgumentException("Unsupported sort mode: " + this); + } + + /** + * Return a {@link SortedDocValues} instance that can be used to sort root documents + * with this mode, the provided values and filters for root/inner documents. + *
+ * For every root document, the values of its inner documents will be aggregated. + *
+ * Allowed Modes: MIN, MAX + *
+ * NOTE: Calling the returned instance on docs that are not root docs is illegal
+ * The returned instance can only be evaluate the current and upcoming docs
+ */
+ public NumericDocValues select(
+ final SortedNumericUnsignedLongValues values,
+ final long missingValue,
+ final BitSet parentDocs,
+ final DocIdSetIterator childDocs,
+ int maxDoc,
+ int maxChildren
+ ) throws IOException {
+ if (parentDocs == null || childDocs == null) {
+ return FieldData.replaceMissing(DocValues.emptyNumeric(), missingValue);
+ }
+
+ return new AbstractNumericDocValues() {
+
+ int lastSeenParentDoc = -1;
+ long lastEmittedValue = missingValue;
+
+ @Override
+ public boolean advanceExact(int parentDoc) throws IOException {
+ assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
+ if (parentDoc == lastSeenParentDoc) {
+ return true;
+ } else if (parentDoc == 0) {
+ lastEmittedValue = missingValue;
+ return true;
+ }
+ final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
+ final int firstChildDoc;
+ if (childDocs.docID() > prevParentDoc) {
+ firstChildDoc = childDocs.docID();
+ } else {
+ firstChildDoc = childDocs.advance(prevParentDoc + 1);
+ }
+
+ lastSeenParentDoc = parentDoc;
+ lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc, maxChildren);
+ return true;
+ }
+
+ @Override
+ public int docID() {
+ return lastSeenParentDoc;
+ }
+
+ @Override
+ public long longValue() {
+ return lastEmittedValue;
+ }
+ };
+ }
+
+ protected long pick(
+ SortedNumericUnsignedLongValues values,
+ long missingValue,
+ DocIdSetIterator docItr,
+ int startDoc,
+ int endDoc,
+ int maxChildren
+ ) throws IOException {
+ throw new IllegalArgumentException("Unsupported sort mode: " + this);
+ }
+
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(this);
@@ -1040,4 +1356,16 @@ public void writeTo(StreamOutput out) throws IOException {
public static MultiValueMode readMultiValueModeFrom(StreamInput in) throws IOException {
return in.readEnum(MultiValueMode.class);
}
+
+ /**
+ * Copied from {@link Long#divideUnsigned(long, long)} and {@link Long#remainderUnsigned(long, long)}
+ */
+ private static long divideUnsignedAndRoundUp(long dividend, long divisor) {
+ assert divisor > 0;
+ final long q = (dividend >>> 1) / divisor << 1;
+ final long r = dividend - q * divisor;
+ final long quotient = q + ((r | ~(r - divisor)) >>> (Long.SIZE - 1));
+ final long rem = r - ((~(r - divisor) >> (Long.SIZE - 1)) & divisor);
+ return quotient + Math.round((double) rem / divisor);
+ }
}
diff --git a/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java b/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java
deleted file mode 100644
index afa562862cfe9..0000000000000
--- a/server/src/test/java/org/opensearch/index/fielddata/fieldcomparator/UnsignedLongMultiValueModeTests.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.fielddata.fieldcomparator;
-
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.FixedBitSet;
-import org.opensearch.common.Numbers;
-import org.opensearch.index.fielddata.AbstractNumericDocValues;
-import org.opensearch.index.fielddata.AbstractSortedNumericDocValues;
-import org.opensearch.test.OpenSearchTestCase;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.RoundingMode;
-import java.util.Arrays;
-import java.util.function.Supplier;
-
-public class UnsignedLongMultiValueModeTests extends OpenSearchTestCase {
- private static FixedBitSet randomRootDocs(int maxDoc) {
- FixedBitSet set = new FixedBitSet(maxDoc);
- for (int i = 0; i < maxDoc; ++i) {
- if (randomBoolean()) {
- set.set(i);
- }
- }
- // the last doc must be a root doc
- set.set(maxDoc - 1);
- return set;
- }
-
- private static FixedBitSet randomInnerDocs(FixedBitSet rootDocs) {
- FixedBitSet innerDocs = new FixedBitSet(rootDocs.length());
- for (int i = 0; i < innerDocs.length(); ++i) {
- if (!rootDocs.get(i) && randomBoolean()) {
- innerDocs.set(i);
- }
- }
- return innerDocs;
- }
-
- public void testSingleValuedLongs() throws Exception {
- final int numDocs = scaledRandomIntBetween(1, 100);
- final long[] array = new long[numDocs];
- final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs);
- for (int i = 0; i < array.length; ++i) {
- if (randomBoolean()) {
- array[i] = randomUnsignedLong().longValue();
- if (docsWithValue != null) {
- docsWithValue.set(i);
- }
- } else if (docsWithValue != null && randomBoolean()) {
- docsWithValue.set(i);
- }
- }
-
- final Supplier