From 83555e7611322c9e871ce72d82676a32c0dd5136 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Sun, 23 Jun 2024 20:49:27 +0530 Subject: [PATCH 01/16] OnHeap Star Tree Implementation Signed-off-by: Sarthak Aggarwal --- .../index/BaseSingleStarTreeBuilder.java | 660 ++++++++++++++++++ .../lucene/index/StarTreeDocValuesWriter.java | 37 + .../common/inject/util/Modules.java | 2 +- .../compositeindex/datacube/MetricStat.java | 3 +- .../aggregators/MetricStatFieldPair.java | 115 +++ .../aggregators/SumValueAggregator.java | 82 +++ .../startree/aggregators/ValueAggregator.java | 64 ++ .../aggregators/ValueAggregatorFactory.java | 51 ++ .../numerictype/StarTreeNumericType.java | 45 ++ .../StarTreeNumericTypeConverters.java | 31 + .../aggregators/numerictype/package-info.java | 13 + .../startree/aggregators/package-info.java | 13 + .../builder/DocValuesIteratorFactory.java | 38 + .../builder/MultipleTreesBuilder.java | 110 +++ .../builder/OnHeapSingleTreeBuilder.java | 204 ++++++ .../startree/builder/SingleTreeBuilder.java | 33 + .../StarTreeDocValuesIteratorFactory.java | 54 ++ .../startree/builder/package-info.java | 13 + .../datacube/startree/data/DataType.java | 67 ++ .../startree/data/StarTreeDocument.java | 29 + .../datacube/startree/node/package-info.java | 13 + .../startree/utils/StarTreeBuilderUtils.java | 131 ++++ .../datacube/startree/utils/package-info.java | 13 + .../aggregators/MetricStatFieldPairTests.java | 69 ++ .../aggregators/SumValueAggregatorTests.java | 64 ++ .../ValueAggregatorFactoryTests.java | 43 ++ .../BaseSingleStarTreeBuilderTests.java | 213 ++++++ .../builder/OnHeapSingleTreeBuilderTests.java | 264 +++++++ .../StarTreeValuesIteratorFactoryTests.java | 113 +++ 29 files changed, 2585 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java create mode 100644 server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java new file mode 100644 index 0000000000000..7b0957b1328de --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java @@ -0,0 +1,660 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.apache.lucene.index; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; +import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Base class for star-tree builder + */ +public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { + + // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined + public static final String STAR_TREE_CODEC = "startreecodec"; + + private static final Logger logger = LogManager.getLogger(BaseSingleStarTreeBuilder.class); + + public static final int STAR_IN_DOC_VALUES_INDEX = -1; + + protected final String[] dimensionsSplitOrder; + protected final Set skipStarNodeCreationForDimensions; + protected final String[] metrics; + + protected final int numMetrics; + protected final int numDimensions; + protected int numDocs; + protected int totalDocs; + protected int numNodes; + protected final int maxLeafDocuments; + + protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); + + // TODO: This will be initialized with OnHeap / OffHeap Implementations (Commented it's occurrences for now) + // private IndexOutput indexOutput; + protected DocIdSetIterator[] dimensionReaders; + protected DocIdSetIterator[] metricReaders; + + protected ValueAggregator[] valueAggregators; + protected IndexNumericFieldData.NumericType[] numericTypes; + protected DocValuesConsumer docValuesConsumer; + protected DocValuesProducer docValuesProducer; + + private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; + private final StarTreeField starTreeField; + private final StarTreeFieldConfiguration starTreeFieldSpec; + private final List metricStatFieldPairs; + private final MapperService mapperService; + + /** + * Constructor for base star-tree builder + * + * @param starTreeField holds the configuration for the star tree + * @param docValuesProducer helps return the doc values iterator for each type based on field name + * @param docValuesConsumer to consume the new aggregated metrics during flush + * @param state stores the segment state + * @param mapperService helps to find the original type of the field + */ + protected BaseSingleStarTreeBuilder( + StarTreeField starTreeField, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { + + logger.info("Building in base star tree builder"); + + // String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); + // logger.info("Star tree file name : {}", docFileName); + + // indexOutput = state.directory.createOutput(docFileName, state.context); + // CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); + this.mapperService = mapperService; + this.starTreeField = starTreeField; + this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); + this.docValuesConsumer = docValuesConsumer; + this.docValuesProducer = docValuesProducer; + this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); + + List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); + this.numDimensions = dimensionsSplitOrder.size(); + this.dimensionsSplitOrder = new String[numDimensions]; + + this.skipStarNodeCreationForDimensions = new HashSet<>(); + this.totalDocs = state.segmentInfo.maxDoc(); + this.dimensionReaders = new DocIdSetIterator[numDimensions]; + Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); + + for (int i = 0; i < numDimensions; i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + this.dimensionsSplitOrder[i] = dimension; + if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { + this.skipStarNodeCreationForDimensions.add(i); + } + FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); + DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); + dimensionReaders[i] = starTreeDocValuesIteratorFactory.createIterator( + dimensionDocValuesType, + dimensionFieldInfos, + docValuesProducer + ); + } + + this.metricStatFieldPairs = generateMetricStatFieldPairs(); + this.numMetrics = metricStatFieldPairs.size(); + this.metrics = new String[numMetrics]; + this.valueAggregators = new ValueAggregator[numMetrics]; + this.numericTypes = new IndexNumericFieldData.NumericType[numMetrics]; + this.metricReaders = new DocIdSetIterator[numMetrics]; + + int index = 0; + for (MetricStatFieldPair metricStatFieldPair : metricStatFieldPairs) { + metrics[index] = metricStatFieldPair.toFieldName(); + valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricStatFieldPair.getMetricStat()); + + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metrics[index]); + if (fieldMapper instanceof NumberFieldMapper) { + numericTypes[index] = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + } else { + numericTypes[index] = IndexNumericFieldData.NumericType.DOUBLE; + } + // Ignore the column for COUNT aggregation function + if (valueAggregators[index].getAggregationType() != MetricStat.COUNT) { + String metricName = metricStatFieldPair.getField(); + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); + DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); + metricReaders[index] = starTreeDocValuesIteratorFactory.createIterator( + metricDocValuesType, + metricFieldInfos, + docValuesProducer + ); + } + index++; + } + this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); + } + + /** + * Generates the MetricStatFieldPairs for all the metrics on a field + * + * @return list of metric stat mapped with respective fields + */ + public List generateMetricStatFieldPairs() { + List metricStatFieldPairs = new ArrayList<>(); + for (Metric metric : this.starTreeField.getMetrics()) { + for (MetricStat metricType : metric.getMetrics()) { + MetricStatFieldPair metricStatFieldPair = new MetricStatFieldPair(metricType, metric.getField()); + metricStatFieldPairs.add(metricStatFieldPair); + } + } + return metricStatFieldPairs; + } + + /** + * Appends a star-tree document to the star-tree. + * + * @param starTreeDocument star tree document to be appended + */ + public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException; + + /** + * Returns the star-tree document of the given document id in the star-tree. + * + * @param docId Document dd + * @return Star tree document + */ + public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException; + + /** + * Returns the star-tree document of the given document id in the star-tree. + * + * @return Star tree document + */ + public abstract List getStarTreeDocuments() throws IOException; + + /** + * Returns the dimension value of the given document and dimension id in the star-tree. + * + * @param docId Document Id + * @param dimensionId Dimension Id + * @return Dimension value + */ + public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; + + /** + * Sorts and aggregates the star-tree Document in the segment, and returns a star-tree document iterator for all the + * aggregated star-tree document. + * + *

This method reads star-tree document from segment and generates the initial star-tree document for the star-tree. + * + * @param numDocs Number of documents in the segment + * @return Iterator for the aggregated star-tree document + */ + public abstract Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException; + + /** + * Generates aggregated star-tree document for star-node. + * + *

This method will do the following steps: + * + *

    + *
  • Creates a temporary buffer for the given range of documents + *
  • Replaces the value for the given dimension Id to {@code STAR} + *
  • Sorts the starTreeDocument inside the temporary buffer + *
  • Aggregates the starTreeDocument with same dimensions + *
  • Returns an iterator for the aggregated starTreeDocument + *
+ * + * @param startDocId Start document id in the star-tree + * @param endDocId End document id (exclusive) in the star-tree + * @param dimensionId Dimension id of the star-node + * @return Iterator for the aggregated starTreeDocument + */ + public abstract Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException; + + /** + * Returns the next segment star-tree document + */ + protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { + long[] dimensions = getNextSegmentStarTreeDocumentDimensions(); + Object[] metrics = getNextSegmentStarTreeDocumentMetrics(); + return new StarTreeDocument(dimensions, metrics); + } + + /** + * Returns the next segment star-tree document for the dimensions + * + * @return dimension values for each of the star-tree dimension + * @throws IOException when we are unable to iterate to the next doc + */ + long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { + long[] dimensions = new long[numDimensions]; + for (int i = 0; i < numDimensions; i++) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + } + + if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { + dimensions[i] = handleDateDimension( + dimensionsSplitOrder[i], + starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]) + ); + } else { + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + } + } + return dimensions; + } + + /** + * Returns the next segment star-tree document for the metrics + * + * @return metric values for each of the star-tree metric + * @throws IOException when we are unable to iterate to the next doc + */ + private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + // Ignore the column for COUNT aggregation function + if (metricReaders[i] != null) { + try { + metricReaders[i].nextDoc(); + } catch (IOException e) { + // TODO : handle null values in columns + logger.error("unable to iterate to next doc", e); + } + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricReaders[i]); + } + } + return metrics; + } + + /** + * Merges a segment star-tree document (raw) into the aggregated star-tree document. + * + *

Will create a new aggregated star-tree document if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated star-tree document + * @param segmentStarTreeDocument Segment star-tree document + * @return Merged starTreeDocument + */ + protected StarTreeDocument aggregateSegmentStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument segmentStarTreeDocument + ) { + // TODO: HANDLE KEYWORDS LATER! + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(segmentStarTreeDocument.dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + try { + StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); + metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentStarTreeDocument.metrics[i], numericType); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot parse initial aggregated value", e); + throw new IllegalArgumentException( + "Cannot parse initial aggregated value [" + segmentStarTreeDocument.metrics[i] + "]" + ); + } + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + try { + StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + aggregatedStarTreeDocument.metrics[i], + (Long) segmentStarTreeDocument.metrics[i], + numericType + ); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot apply segment raw value", e); + throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentStarTreeDocument.metrics[i] + "]"); + } + } + return aggregatedStarTreeDocument; + } + } + + /** + * Merges a star-tree document (aggregated) into the aggregated document. + * + *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated star-tree document + * @param starTreeStarTreeDocument Star-tree document + * @return Merged star-tree document + */ + public StarTreeDocument aggregateStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument starTreeStarTreeDocument + ) { + // aggregate the documents + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument.dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + try { + metrics[i] = valueAggregators[i].cloneAggregatedValue(starTreeStarTreeDocument.metrics[i]); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot clone aggregated value", e); + throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + } + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + try { + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( + starTreeStarTreeDocument.metrics[i], + aggregatedStarTreeDocument.metrics[i] + ); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot apply aggregated value", e); + throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + } + } + return aggregatedStarTreeDocument; + } + } + + // TODO: This will be taken care in off heap implementation for merges + // public abstract void build(List starTreeValues) throws IOException; + + public void build() throws IOException { + long startTime = System.currentTimeMillis(); + logger.info("Tree of Aggregations build is a go with config {}", starTreeField); + + Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalDocs); + logger.info("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + build(starTreeDocumentIterator); + logger.info("Finished Building TOA in ms : {}", (System.currentTimeMillis() - startTime)); + } + + /** + * Builds the star tree using Star-Tree Document + * + * @param starTreeDocumentIterator contains the sorted and aggregated documents + * @throws IOException when we are unable to build star-tree + */ + public void build(Iterator starTreeDocumentIterator) throws IOException { + int numSegmentStarTreeDocument = totalDocs; + + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + int numStarTreeDocument = numDocs; + logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + + if (numDocs == 0) { + // TODO: Uncomment when segment codec is ready + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + return; + } + + constructStarTree(rootNode, 0, numDocs); + int numStarTreeDocumentUnderStarNode = numDocs - numStarTreeDocument; + logger.info( + "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", + numNodes, + numStarTreeDocumentUnderStarNode + ); + + createAggregatedDocs(rootNode); + int numAggregatedStarTreeDocument = numDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.info("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + + // Create doc values indices in disk + // TODO: Uncomment when segment codec is ready + // createSortedDocValuesIndices(docValuesConsumer); + + // Serialize and save in disk + // TODO: Uncomment when segment codec is ready + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + + // TODO: Write star tree metadata for off heap implementation + + } + + /** + * Appends a starTreeDocument to star tree + * + * @param starTreeDocument star-tree document + * @throws IOException throws an exception if we are unable to append the doc + */ + private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { + appendStarTreeDocument(starTreeDocument); + numDocs++; + } + + /** + * Returns a new node + * + * @return return new star-tree node + */ + private StarTreeBuilderUtils.TreeNode getNewNode() { + numNodes++; + return new StarTreeBuilderUtils.TreeNode(); + } + + /** + * Implements the algorithm to construct a star-tree based on star-tree documents + * + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id + * @throws IOException throws an exception if we are unable to construct the tree + */ + private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { + + int childDimensionId = node.dimensionId + 1; + if (childDimensionId == numDimensions) { + return; + } + + // Construct all non-star children nodes + node.childDimensionId = childDimensionId; + Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); + node.children = children; + + // Construct star-node if required + if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) { + children.put((long) StarTreeBuilderUtils.ALL, constructStarNode(startDocId, endDocId, childDimensionId)); + } + + // Further split on child nodes if required + for (StarTreeBuilderUtils.TreeNode child : children.values()) { + if (child.endDocId - child.startDocId > maxLeafDocuments) { + constructStarTree(child, child.startDocId, child.endDocId); + } + } + } + + /** + * Constructs non star tree nodes + * + * @param startDocId start document id + * @param endDocId end document id + * @param dimensionId id of the dimension in the star tree + * @return root node with non-star nodes constructed + * @throws IOException throws an exception if we are unable to construct non-star nodes + */ + private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) + throws IOException { + Map nodes = new HashMap<>(); + int nodeStartDocId = startDocId; + long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); + for (int i = startDocId + 1; i < endDocId; i++) { + long dimensionValue = getDimensionValue(i, dimensionId); + if (dimensionValue != nodeDimensionValue) { + StarTreeBuilderUtils.TreeNode child = getNewNode(); + child.dimensionId = dimensionId; + child.dimensionValue = nodeDimensionValue; + child.startDocId = nodeStartDocId; + child.endDocId = i; + nodes.put(nodeDimensionValue, child); + + nodeStartDocId = i; + nodeDimensionValue = dimensionValue; + } + } + StarTreeBuilderUtils.TreeNode lastNode = getNewNode(); + lastNode.dimensionId = dimensionId; + lastNode.dimensionValue = nodeDimensionValue; + lastNode.startDocId = nodeStartDocId; + lastNode.endDocId = endDocId; + nodes.put(nodeDimensionValue, lastNode); + return nodes; + } + + /** + * Constructs star tree nodes + * + * @param startDocId start document id + * @param endDocId end document id + * @param dimensionId id of the dimension in the star tree + * @return root node with star nodes constructed + * @throws IOException throws an exception if we are unable to construct non-star nodes + */ + private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + StarTreeBuilderUtils.TreeNode starNode = getNewNode(); + starNode.dimensionId = dimensionId; + starNode.dimensionValue = StarTreeBuilderUtils.ALL; + starNode.startDocId = numDocs; + Iterator starTreeDocumentIterator = generateStarTreeForStarNode(startDocId, endDocId, dimensionId); + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + starNode.endDocId = numDocs; + return starNode; + } + + /** + * Returns aggregated star-tree document + * + * @param node star-tree node + * @return aggregated star-tree documents + * @throws IOException throws an exception upon failing to create new aggregated docs based on star tree + */ + private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node) throws IOException { + StarTreeDocument aggregatedStarTreeDocument = null; + if (node.children == null) { + // For leaf node + + if (node.startDocId == node.endDocId - 1) { + // If it has only one document, use it as the aggregated document + aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId); + node.aggregatedDocId = node.startDocId; + } else { + // If it has multiple documents, aggregate all of them + for (int i = node.startDocId; i < node.endDocId; i++) { + aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, getStarTreeDocument(i)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } else { + // For non-leaf node + if (node.children.containsKey((long) StarTreeBuilderUtils.ALL)) { + // If it has star child, use the star child aggregated document directly + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + if (child.dimensionValue == StarTreeBuilderUtils.ALL) { + aggregatedStarTreeDocument = createAggregatedDocs(child); + node.aggregatedDocId = child.aggregatedDocId; + } else { + createAggregatedDocs(child); + } + } + } else { + // If no star child exists, aggregate all aggregated documents from non-star children + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, createAggregatedDocs(child)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } + return aggregatedStarTreeDocument; + } + + /** + * Handles the dimension of date time field type + * + * @param fieldName name of the field + * @param val value of the field + * @return returns the converted dimension of the field to a particular granularity + */ + private long handleDateDimension(final String fieldName, final long val) { + // TODO: handle timestamp granularity + return val; + } + + public void close() throws IOException { + // boolean success = false; + // try { + // if (indexOutput != null) { + // indexOutput.writeInt(-1); + // CodecUtil.writeFooter(indexOutput); // write checksum + // } + // success = true; + // } catch (Exception e) { + // throw new RuntimeException(e); + // } finally { + // if (success) { + // IOUtils.close(indexOutput); + // } else { + // IOUtils.closeWhileHandlingException(indexOutput); + // } + // indexOutput = null; + // } + } + +} diff --git a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java new file mode 100644 index 0000000000000..e8d96226da2ee --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +/** + * A wrapper for {@link DocValuesWriter} that contains the {@link DocValuesType} of the doc + */ +public class StarTreeDocValuesWriter { + + private final DocValuesType docValuesType; + private final DocValuesWriter docValuesWriter; + + public StarTreeDocValuesWriter(DocValuesType docValuesType, DocValuesWriter docValuesWriter) { + this.docValuesType = docValuesType; + this.docValuesWriter = docValuesWriter; + } + + /** + * Get the doc values type + */ + public DocValuesType getDocValuesType() { + return docValuesType; + } + + /** + * Get the doc values writer + */ + public DocValuesWriter getDocValuesWriter() { + return docValuesWriter; + } +} diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index ae37cb3d29a68..b5a5a83ac3af9 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -202,7 +202,7 @@ public Void visit(Binding binding) { if (!overriddenKeys.remove(binding.getKey())) { super.visit(binding); - // Record when a scope instance is used in a binding + // record when a scope instance is used in a binding Scope scope = getScopeInstanceOrNull(binding); if (scope != null) { scopeInstancesInUse.put(scope, binding.getSource()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java index fbde296b15f7e..a5a6d07ad9bbc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -21,7 +21,8 @@ public enum MetricStat { AVG("avg"), SUM("sum"), MIN("min"), - MAX("max"); + MAX("max"), + UNSUPPORTED("unsupported"); private final String typeName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java new file mode 100644 index 0000000000000..3fb041373ff51 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; + +import java.util.Comparator; + +/** + * Builds aggregation function and doc values field pair to support various aggregations + * @opensearch.experimental + */ +public class MetricStatFieldPair implements Comparable { + + public static final String DELIMITER = "__"; + public static final String STAR = "*"; + public static final MetricStatFieldPair COUNT_STAR = new MetricStatFieldPair(MetricStat.COUNT, STAR); + + private final MetricStat metricStat; + private final String field; + + /** + * Constructor for MetricStatFieldPair + */ + public MetricStatFieldPair(MetricStat metricStat, String field) { + this.metricStat = metricStat; + if (metricStat == MetricStat.COUNT) { + this.field = STAR; + } else { + this.field = field; + } + } + + /** + * @return Metric Type + */ + public MetricStat getMetricStat() { + return metricStat; + } + + /** + * @return field Name + */ + public String getField() { + return field; + } + + /** + * @return field name with metric type and field + */ + public String toFieldName() { + return toFieldName(metricStat, field); + } + + /** + * Builds field name with metric type and field + */ + public static String toFieldName(MetricStat metricType, String field) { + return metricType.getTypeName() + DELIMITER + field; + } + + /** + * Builds MetricStatFieldPair from field name + */ + public static MetricStatFieldPair fromFieldName(String fieldName) { + String[] parts = fieldName.split(DELIMITER, 2); + return fromMetricAndFieldName(parts[0], parts[1]); + } + + /** + * Builds MetricStatFieldPair from metric and field name + */ + private static MetricStatFieldPair fromMetricAndFieldName(String metricName, String fieldName) { + MetricStat metricType = MetricStat.fromTypeName(metricName); + if (metricType == MetricStat.COUNT) { + return COUNT_STAR; + } else { + return new MetricStatFieldPair(metricType, fieldName); + } + } + + @Override + public int hashCode() { + return 31 * metricStat.hashCode() + field.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof MetricStatFieldPair) { + MetricStatFieldPair anotherPair = (MetricStatFieldPair) obj; + return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); + } + return false; + } + + @Override + public String toString() { + return toFieldName(); + } + + @Override + public int compareTo(MetricStatFieldPair other) { + return Comparator.comparing((MetricStatFieldPair o) -> o.field) + .thenComparing((MetricStatFieldPair o) -> o.metricStat) + .compare(this, other); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java new file mode 100644 index 0000000000000..a3a14f9bbb11a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.search.aggregations.metrics.CompensatedSum; + +/** + * Sum value aggregator for star tree + * + * @opensearch.internal + */ +public class SumValueAggregator implements ValueAggregator { + public static final DataType AGGREGATED_VALUE_TYPE = DataType.DOUBLE; + + @Override + public MetricStat getAggregationType() { + return MetricStat.SUM; + } + + @Override + public DataType getAggregatedValueType() { + return AGGREGATED_VALUE_TYPE; + } + + @Override + public Double getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType) { + return starTreeNumericType.getDoubleValue(rawValue); + } + + @Override + public Double applySegmentRawValue(Double value, Long rawValue, StarTreeNumericType starTreeNumericType) { + CompensatedSum kahanSummation = new CompensatedSum(0, 0); + kahanSummation.add(value); + kahanSummation.add(starTreeNumericType.getDoubleValue(rawValue)); + return kahanSummation.value(); + } + + @Override + public Double applyAggregatedValue(Double value, Double aggregatedValue) { + CompensatedSum kahanSummation = new CompensatedSum(0, 0); + kahanSummation.add(value); + kahanSummation.add(aggregatedValue); + return kahanSummation.value(); + } + + @Override + public Double cloneAggregatedValue(Double value) { + return value; + } + + @Override + public int getMaxAggregatedValueByteSize() { + return Double.BYTES; + } + + @Override + public Long convertAggregationTypeToSortableLongValue(Double value) { + try { + return NumericUtils.doubleToSortableLong(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } + } + + @Override + public Double convertSortableLongToAggregatedTypeValue(Long value, StarTreeNumericType type) { + try { + return type.getDoubleValue(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java new file mode 100644 index 0000000000000..b9fa3d5b69a3f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; + +/** + * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. + * @opensearch.experimental + */ +public interface ValueAggregator { + + /** + * Returns the type of the aggregation. + */ + MetricStat getAggregationType(); + + /** + * Returns the data type of the aggregated value. + */ + DataType getAggregatedValueType(); + + /** + * Returns the initial aggregated value. + */ + A getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType); + + /** + * Applies a raw value to the current aggregated value. + */ + A applySegmentRawValue(A value, Long rawValue, StarTreeNumericType starTreeNumericType); + + /** + * Applies an aggregated value to the current aggregated value. + */ + A applyAggregatedValue(A value, A aggregatedValue); + + /** + * Clones an aggregated value. + */ + A cloneAggregatedValue(A value); + + /** + * Returns the maximum size in bytes of the aggregated values seen so far. + */ + int getMaxAggregatedValueByteSize(); + + /** + * Converts an aggregated value into a Long type. + */ + Long convertAggregationTypeToSortableLongValue(A value); + + /** + * Converts an aggregated value from a Long type. + */ + A convertSortableLongToAggregatedTypeValue(Long rawValue, StarTreeNumericType type); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java new file mode 100644 index 0000000000000..523b3fbec2e7c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; + +/** + * Value aggregator factory for a given aggregation type + * @opensearch.experimental + */ +public class ValueAggregatorFactory { + private ValueAggregatorFactory() {} + + /** + * Returns a new instance of value aggregator for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Value aggregator + */ + public static ValueAggregator getValueAggregator(MetricStat aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return new SumValueAggregator(); + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } + + /** + * Returns the data type of the aggregated value for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Data type of the aggregated value + */ + public static DataType getAggregatedValueType(MetricStat aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return SumValueAggregator.AGGREGATED_VALUE_TYPE; + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java new file mode 100644 index 0000000000000..ed41445ac5cdf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; + +import org.opensearch.index.fielddata.IndexNumericFieldData; + +import java.util.function.Function; + +public enum StarTreeNumericType { + HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), + FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), + LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), + DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble); + + final IndexNumericFieldData.NumericType numericType; + final Function converter; + + StarTreeNumericType(IndexNumericFieldData.NumericType numericType, Function converter) { + this.numericType = numericType; + this.converter = converter; + } + + public double getDoubleValue(long rawValue) { + return this.converter.apply(rawValue); + } + + public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericType numericType) { + switch (numericType) { + case HALF_FLOAT: + return StarTreeNumericType.HALF_FLOAT; + case FLOAT: + return StarTreeNumericType.FLOAT; + case LONG: + return StarTreeNumericType.LONG; + default: + return StarTreeNumericType.DOUBLE; + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java new file mode 100644 index 0000000000000..59a2c3418e931 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.util.NumericUtils; + +public class StarTreeNumericTypeConverters { + + public static double halfFloatPointToDouble(Long value) { + return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue()); + } + + public static double floatPointToDouble(Long value) { + return NumericUtils.sortableIntToFloat((int) value.longValue()); + } + + public static double longToDouble(Long value) { + return (double) value; + } + + public static Double sortableLongtoDouble(Long value) { + return NumericUtils.sortableLongToDouble(value); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java new file mode 100644 index 0000000000000..b9a9bb8f1427d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Numeric Types for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java new file mode 100644 index 0000000000000..27565ffded2cf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Aggregators for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java new file mode 100644 index 0000000000000..66302f84f1449 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * An interface to support iterators for various doc values types. + * @opensearch.experimental + */ +public interface DocValuesIteratorFactory { + + /** + * Creates an iterator for the given doc values type and field using the doc values producer + */ + DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; + + /** + * Returns the next value for the given iterator + */ + long getNextValue(DocIdSetIterator iterator) throws IOException; + + /** + * Returns the doc id for the next document from the given iterator + */ + int nextDoc(DocIdSetIterator iterator) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java new file mode 100644 index 0000000000000..ad3d33474840f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.MapperService; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class MultipleTreesBuilder implements Closeable { + + private static final Logger logger = LogManager.getLogger(MultipleTreesBuilder.class); + + private final List starTreeFields; + private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; + private final DocValuesConsumer docValuesConsumer; + private final SegmentWriteState state; + private final DocValuesProducer docValuesProducer; + private final MapperService mapperService; + + public MultipleTreesBuilder( + List starTreeFields, + StarTreeFieldConfiguration.StarTreeBuildMode buildMode, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) { + this.starTreeFields = starTreeFields; + if (starTreeFields == null || starTreeFields.isEmpty()) { + throw new IllegalArgumentException("Must provide star-tree builder configs"); + } + this.buildMode = buildMode; + this.docValuesProducer = docValuesProducer; + this.docValuesConsumer = docValuesConsumer; + this.state = segmentWriteState; + this.mapperService = mapperService; + } + + /** + * Builds the star-trees. + */ + public void build() throws Exception { + long startTime = System.currentTimeMillis(); + int numStarTrees = starTreeFields.size(); + logger.info("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + + // Build all star-trees + for (int i = 0; i < numStarTrees; i++) { + StarTreeField starTreeField = starTreeFields.get(i); + try ( + SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( + starTreeField, + buildMode, + docValuesProducer, + docValuesConsumer, + state, + mapperService + ) + ) { + singleTreeBuilder.build(); + } + } + logger.info( + "Took {} ms to building {} star-trees with configs: {} using {} builder", + System.currentTimeMillis() - startTime, + numStarTrees, + starTreeFields, + buildMode + ); + } + + @Override + public void close() throws IOException { + + } + + private static SingleTreeBuilder getSingleTreeBuilder( + StarTreeField starTreeField, + StarTreeFieldConfiguration.StarTreeBuildMode buildMode, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { + switch (buildMode) { + case ON_HEAP: + return new OnHeapSingleTreeBuilder(starTreeField, docValuesProducer, docValuesConsumer, state, mapperService); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java new file mode 100644 index 0000000000000..6baaedb1d0710 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -0,0 +1,204 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * On heap single tree builder + */ +public class OnHeapSingleTreeBuilder extends BaseSingleStarTreeBuilder { + + private final List starTreeDocuments = new ArrayList<>(); + + /** + * Constructor for OnHeapSingleTreeBuilder + * + * @param starTreeField star-tree field + * @param docValuesProducer document values producer + * @param docValuesConsumer document values consumer + * @param segmentWriteState segment write state + * @param mapperService + * @throws IOException throws an exception we are unable to construct an onheap star-tree + */ + public OnHeapSingleTreeBuilder( + StarTreeField starTreeField, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) throws IOException { + super(starTreeField, docValuesProducer, docValuesConsumer, segmentWriteState, mapperService); + } + + @Override + public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException { + starTreeDocuments.add(starTreeDocument); + } + + @Override + public StarTreeDocument getStarTreeDocument(int docId) throws IOException { + return starTreeDocuments.get(docId); + } + + @Override + public List getStarTreeDocuments() throws IOException { + return starTreeDocuments; + } + + // TODO: should this be just long? + @Override + public long getDimensionValue(int docId, int dimensionId) throws IOException { + return starTreeDocuments.get(docId).dimensions[dimensionId]; + } + + // Handles star-tree rebuilds during merges :) + // @Override + // public void build(List starTreeValues) throws IOException { + // TODO: This will be handled during off heap implementation for merges + // } + + @Override + public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; + for (int i = 0; i < numDocs; i++) { + starTreeDocuments[i] = getNextSegmentStarTreeDocument(); + } + return processStarTreeDocuments(starTreeDocuments); + } + + /** + * Sort, aggregates and merges the star-tree documents + * @param starTreeDocuments star-tree documents + * @return iterator for star-tree documents + * @throws IOException throws when unable to sort, merge and aggregate star-tree documents + */ + public Iterator processStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + + // sort the documents + Arrays.sort(starTreeDocuments, (o1, o2) -> { + for (int i = 0; i < numDimensions; i++) { + if (o1.dimensions[i] != o2.dimensions[i]) { + return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + } + } + return 0; + }); + + // merge the documents + return mergeStarTreeDocuments(starTreeDocuments); + } + + /** + * Merges the star-tree documents + * @param starTreeDocuments star-tree documents + * @return iterator to aggregate star-tree documents + */ + private Iterator mergeStarTreeDocuments(StarTreeDocument[] starTreeDocuments) { + return new Iterator<>() { + boolean hasNext = true; + StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; + int docId = 1; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public StarTreeDocument next() { + // aggregate as we move on to the next doc + StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + while (docId < starTreeDocuments.length) { + StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; + if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { + currentStarTreeDocument = starTreeDocument; + return next; + } else { + next = aggregateStarTreeDocument(next, starTreeDocument); + } + } + hasNext = false; + return next; + } + }; + } + + /** + * Generates a star-tree for a given star-node + * @param startDocId Start document id in the star-tree + * @param endDocId End document id (exclusive) in the star-tree + * @param dimensionId Dimension id of the star-node + * @return iterator for star-tree documents of star-node + * @throws IOException throws when unable to generate star-tree for star-node + */ + @Override + public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + int numDocs = endDocId - startDocId; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; + for (int i = 0; i < numDocs; i++) { + starTreeDocuments[i] = getStarTreeDocument(startDocId + i); + } + Arrays.sort(starTreeDocuments, (o1, o2) -> { + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (o1.dimensions[i] != o2.dimensions[i]) { + return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + } + } + return 0; + }); + return new Iterator() { + boolean hasNext = true; + StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; + int docId = 1; + + private boolean hasSameDimensions(StarTreeDocument starTreeDocument1, StarTreeDocument starTreeDocument2) { + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (starTreeDocument1.dimensions[i] != starTreeDocument2.dimensions[i]) { + return false; + } + } + return true; + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public StarTreeDocument next() { + StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; + while (docId < numDocs) { + StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; + if (!hasSameDimensions(starTreeDocument, currentStarTreeDocument)) { + currentStarTreeDocument = starTreeDocument; + return next; + } else { + next = aggregateStarTreeDocument(next, starTreeDocument); + } + } + hasNext = false; + return next; + } + }; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java new file mode 100644 index 0000000000000..3b26f4c5e04c3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import java.io.Closeable; +import java.io.IOException; + +/** + * A star-tree builder that builds a single star-tree. + * @opensearch.experimental + */ +public interface SingleTreeBuilder extends Closeable { + + /** + * Builds the star tree based on star-tree field + * @throws IOException when we are unable to build star-tree + */ + void build() throws Exception; + + /** + * Builds the star tree using star-tree document values during segment merges + * @param starTreeValues star-tree document values + * @throws IOException when we are unable to build star-tree + */ + // void build(List starTreeValues) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java new file mode 100644 index 0000000000000..0105ff42045de --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * A factory class to return respective doc values iterator based on the doc volues type. + * @opensearch.experimental + */ +public class StarTreeDocValuesIteratorFactory implements DocValuesIteratorFactory { + + @Override + public DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + switch (type) { + case SORTED_SET: + return producer.getSortedSet(field); + case SORTED_NUMERIC: + return producer.getSortedNumeric(field); + default: + throw new IllegalArgumentException("Unsupported DocValuesType: " + type); + } + } + + @Override + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedSetDocValues) { + return ((SortedSetDocValues) iterator).nextOrd(); + } else if (iterator instanceof SortedNumericDocValues) { + return ((SortedNumericDocValues) iterator).nextValue(); + } else { + throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); + } + } + + @Override + public int nextDoc(DocIdSetIterator iterator) throws IOException { + return iterator.nextDoc(); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java new file mode 100644 index 0000000000000..80eed545ef8a5 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Builders for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java new file mode 100644 index 0000000000000..3c737c44b7219 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.data; + +/** + * Data type of doc values + * @opensearch.internal + */ +public enum DataType { + INT(Integer.BYTES, true), + LONG(Long.BYTES, true), + FLOAT(Float.BYTES, true), + DOUBLE(Double.BYTES, true); + + private final int size; + private final boolean numeric; + + DataType(int size, boolean numeric) { + this.size = size; + this.numeric = numeric; + } + + /** + * Returns the number of bytes needed to store the data type. + */ + public int size() { + if (size >= 0) { + return size; + } + throw new IllegalStateException("Cannot get number of bytes for: " + this); + } + + /** + * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), + * {@code false} otherwise. + */ + public boolean isNumeric() { + return numeric; + } + + /** + * Converts the given string value to the data type. Returns byte[] for BYTES. + */ + public Object convert(String value) { + try { + switch (this) { + case INT: + return Integer.valueOf(value); + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + default: + throw new IllegalStateException(); + } + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java new file mode 100644 index 0000000000000..cb4253e21c141 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.data; + +import java.util.Arrays; + +/** + * Star tree document + */ +public class StarTreeDocument { + public final long[] dimensions; + public final Object[] metrics; + + public StarTreeDocument(long[] dimensions, Object[] metrics) { + this.dimensions = dimensions; + this.metrics = metrics; + } + + @Override + public String toString() { + return Arrays.toString(dimensions) + " | " + Arrays.toString(metrics); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java new file mode 100644 index 0000000000000..3af5020b76da2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Node for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java new file mode 100644 index 0000000000000..9fa980277f27b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Util class for building star tree + * @opensearch.experimental + */ +public class StarTreeBuilderUtils { + + private static final Logger logger = LogManager.getLogger(StarTreeBuilderUtils.class); + + // TODO: To be moved to off heap star tree implementation + public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; + public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; + public static final long SERIALIZABLE_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES + * NUM_LONG_SERIALIZABLE_FIELDS); + + private StarTreeBuilderUtils() {} + + public static final int ALL = -1; + public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; + + /** Tree node representation */ + public static class TreeNode { + public int dimensionId = ALL; + public long dimensionValue = ALL; + public int startDocId = ALL; + public int endDocId = ALL; + public int aggregatedDocId = ALL; + public int childDimensionId = ALL; + public Map children; + } + + /** Serializes the tree */ + public static void serializeTree(IndexOutput indexOutput, TreeNode rootNode, String[] dimensions, int numNodes) throws IOException { + int headerSizeInBytes = computeHeaderByteSize(dimensions); + long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_SIZE_IN_BYTES; + + logger.info("Star tree size in bytes : {}", totalSizeInBytes); + + writeHeader(indexOutput, headerSizeInBytes, dimensions, numNodes); + writeNodes(indexOutput, rootNode); + } + + /** Computes the size of the header for the tree */ + static int computeHeaderByteSize(String[] dimensions) { + // Magic marker (8), version (4), size of header (4) and number of dimensions (4) + int headerSizeInBytes = 20; + + for (String dimension : dimensions) { + headerSizeInBytes += Integer.BYTES; // For dimension index + headerSizeInBytes += Integer.BYTES; // For length of dimension name + headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name + } + + headerSizeInBytes += Integer.BYTES; // For number of nodes. + return headerSizeInBytes; + } + + /** Writes the header of the tree */ + static void writeHeader(IndexOutput output, int headerSizeInBytes, String[] dimensions, int numNodes) throws IOException { + output.writeLong(MAGIC_MARKER); + output.writeInt(1); + output.writeInt(headerSizeInBytes); + output.writeInt(dimensions.length); + for (int i = 0; i < dimensions.length; i++) { + output.writeInt(i); + output.writeString(dimensions[i]); + } + output.writeInt(numNodes); + } + + /** Writes the nodes of the tree */ + static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException { + Queue queue = new LinkedList<>(); + queue.add(rootNode); + + int currentNodeId = 0; + while (!queue.isEmpty()) { + TreeNode node = queue.remove(); + + if (node.children == null) { + writeNode(output, node, ALL, ALL); + } else { + // Sort all children nodes based on dimension value + List sortedChildren = new ArrayList<>(node.children.values()); + sortedChildren.sort(Comparator.comparingLong(o -> o.dimensionValue)); + + int firstChildId = currentNodeId + queue.size() + 1; + int lastChildId = firstChildId + sortedChildren.size() - 1; + writeNode(output, node, firstChildId, lastChildId); + + queue.addAll(sortedChildren); + } + + currentNodeId++; + } + } + + /** Writes a node of the tree */ + private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException { + output.writeInt(node.dimensionId); + output.writeLong(node.dimensionValue); + output.writeInt(node.startDocId); + output.writeInt(node.endDocId); + output.writeInt(node.aggregatedDocId); + output.writeInt(firstChildId); + output.writeInt(lastChildId); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java new file mode 100644 index 0000000000000..92930de98970d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Utility to support Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java new file mode 100644 index 0000000000000..fd026f49c0140 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.test.OpenSearchTestCase; + +public class MetricStatFieldPairTests extends OpenSearchTestCase { + + public void testConstructor() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(MetricStat.SUM, pair.getMetricStat()); + assertEquals("column1", pair.getField()); + } + + public void testCountStarConstructor() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.COUNT, "anything"); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + } + + public void testToFieldName() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.AVG, "column2"); + assertEquals("avg__column2", pair.toFieldName()); + } + + public void testFromFieldName() { + MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("max__column3"); + assertEquals(MetricStat.MAX, pair.getMetricStat()); + assertEquals("column3", pair.getField()); + } + + public void testCountStarFromFieldName() { + MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("count__*"); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + assertSame(MetricStatFieldPair.COUNT_STAR, pair); + } + + public void testEquals() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(pair1, pair2); + assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.AVG, "column1")); + assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.SUM, "column2")); + } + + public void testHashCode() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(pair1.hashCode(), pair2.hashCode()); + } + + public void testCompareTo() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column2"); + MetricStatFieldPair pair3 = new MetricStatFieldPair(MetricStat.AVG, "column1"); + assertTrue(pair1.compareTo(pair2) < 0); + assertTrue(pair2.compareTo(pair1) > 0); + assertTrue(pair1.compareTo(pair3) > 0); + assertTrue(pair3.compareTo(pair1) < 0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java new file mode 100644 index 0000000000000..58709934c894e --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.test.OpenSearchTestCase; + +public class SumValueAggregatorTests extends OpenSearchTestCase { + + private final SumValueAggregator aggregator = new SumValueAggregator(); + + public void testGetAggregationType() { + assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValue(1L, StarTreeNumericType.LONG), 0.0); + assertThrows(NullPointerException.class, () -> aggregator.getInitialAggregatedValue(null, StarTreeNumericType.DOUBLE)); + } + + public void testApplySegmentRawValue() { + assertEquals(5.0, aggregator.applySegmentRawValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + assertThrows(NullPointerException.class, () -> aggregator.applySegmentRawValue(3.14, null, StarTreeNumericType.DOUBLE)); + } + + public void testApplyAggregatedValue() { + assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); + } + + public void testCloneAggregatedValue() { + assertEquals(3.14, aggregator.cloneAggregatedValue(3.14), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testConvertAggregationTypeToSortableLongValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.convertAggregationTypeToSortableLongValue(3.14), 0.0); + } + + public void testConvertSortableLongToAggregatedTypeValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals( + NumericUtils.sortableLongToDouble(3L), + aggregator.convertSortableLongToAggregatedTypeValue(3L, StarTreeNumericType.DOUBLE), + 0.0 + ); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java new file mode 100644 index 0000000000000..f80783dacb643 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.test.OpenSearchTestCase; + +public class ValueAggregatorFactoryTests extends OpenSearchTestCase { + + public void testGetValueAggregatorForSumType() { + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM); + assertNotNull(aggregator); + assertEquals(SumValueAggregator.class, aggregator.getClass()); + } + + public void testGetValueAggregatorForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getValueAggregator(MetricStat.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } + + public void testGetAggregatedValueTypeForSumType() { + DataType dataType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, dataType); + } + + public void testGetAggregatedValueTypeForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getAggregatedValueType(MetricStat.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java new file mode 100644 index 0000000000000..82be99faa8bd8 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java @@ -0,0 +1,213 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BaseSingleStarTreeBuilderTests extends OpenSearchTestCase { + + private static BaseSingleStarTreeBuilder builder; + private static MapperService mapperService; + private static List dimensionsOrder; + private static List fields = List.of( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10" + ); + private static List metrics; + private static Directory directory; + private static FieldInfo[] fieldsInfo; + + @BeforeClass + public static void setup() throws IOException { + + dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + + StarTreeField compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + ); + DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + final SegmentWriteState state = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + builder = new BaseSingleStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + @Override + public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} + + @Override + public StarTreeDocument getStarTreeDocument(int docId) throws IOException { + return null; + } + + @Override + public List getStarTreeDocuments() throws IOException { + return List.of(); + } + + @Override + public long getDimensionValue(int docId, int dimensionId) throws IOException { + return 0; + } + + @Override + public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + return null; + } + + @Override + public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException { + return null; + } + }; + } + + public void test_generateMetricStatFieldPairs() throws IOException { + List metricStatFieldPairs = builder.generateMetricStatFieldPairs(); + List expectedMetricStatFieldPairs = List.of( + new MetricStatFieldPair(MetricStat.SUM, "field2"), + new MetricStatFieldPair(MetricStat.SUM, "field4") + ); + assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); + } + + public void test_aggregateStarTreeDocument() { + StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); + StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + + StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); + StarTreeDocument mergedStarTreeDocument = builder.aggregateStarTreeDocument(starTreeDocument1, starTreeDocument2); + + assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); + assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java new file mode 100644 index 0000000000000..e81c836c9d723 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java @@ -0,0 +1,264 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { + + private static OnHeapSingleTreeBuilder builder; + private static MapperService mapperService; + private static List dimensionsOrder; + private static List fields = List.of( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10" + ); + private static List metrics; + private static Directory directory; + private static FieldInfo[] fieldsInfo; + + @BeforeClass + public static void setup() throws IOException { + dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + + DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + StarTreeField compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + ); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + final SegmentWriteState writeState = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, writeState, mapperService); + } + + public void test_processStarTreeDocuments() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + int numOfAggregatedDocuments = 0; + while (starTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_processStarTreeDocuments_nullDocument() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + + StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + assertThrows("Cannot apply aggregated value [null]", IllegalArgumentException.class, starTreeDocumentIterator::next); + + } + + public void test_build() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + builder.build(starTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + List expectedStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), + new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) + ); + Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator(); + Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); + while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java new file mode 100644 index 0000000000000..7f6978dbf2720 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; + +import org.mockito.Mockito; + +import static org.mockito.Mockito.when; + +public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase { + + private static StarTreeDocValuesIteratorFactory factory; + private static FieldInfo mockFieldInfo; + + @BeforeClass + public static void setup() { + factory = new StarTreeDocValuesIteratorFactory(); + mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + + public void testCreateIterator_SortedSet() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_SortedNumeric() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_UnsupportedType() { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + factory.createIterator(DocValuesType.BINARY, mockFieldInfo, producer); + }); + assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); + } + + public void testGetNextValue_SortedSet() throws IOException { + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(iterator.nextOrd()).thenReturn(42L); + + long result = factory.getNextValue(iterator); + assertEquals(42L, result); + } + + public void testGetNextValue_SortedNumeric() throws IOException { + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(iterator.nextValue()).thenReturn(123L); + + long result = factory.getNextValue(iterator); + assertEquals(123L, result); + } + + public void testGetNextValue_UnsupportedIterator() { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); + } + + public void testNextDoc() throws IOException { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + when(iterator.nextDoc()).thenReturn(5); + + int result = factory.nextDoc(iterator); + assertEquals(5, result); + } +} From 9d8b6c6367a7b4ac20548310ea3649419c3cd27c Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 24 Jun 2024 01:01:10 +0530 Subject: [PATCH 02/16] addressed nits Signed-off-by: Sarthak Aggarwal --- ...eBuilder.java => BaseStarTreeBuilder.java} | 188 +++++++----------- .../common/inject/util/Modules.java | 3 +- .../aggregators/SumValueAggregator.java | 2 +- .../startree/aggregators/ValueAggregator.java | 2 +- .../numerictype/StarTreeNumericType.java | 4 +- ...ory.java => DocValuesIteratorAdapter.java} | 4 +- .../builder/OnHeapSingleTreeBuilder.java | 24 +-- .../startree/builder/SingleTreeBuilder.java | 8 - ... => StarTreeDocValuesIteratorAdapter.java} | 4 +- ...reesBuilder.java => StarTreesBuilder.java} | 10 +- .../startree/utils/StarTreeBuilderUtils.java | 101 +--------- .../aggregators/SumValueAggregatorTests.java | 4 +- ...sts.java => BaseStarTreeBuilderTests.java} | 14 +- .../StarTreeValuesIteratorFactoryTests.java | 10 +- 14 files changed, 117 insertions(+), 261 deletions(-) rename server/src/main/java/org/apache/lucene/index/{BaseSingleStarTreeBuilder.java => BaseStarTreeBuilder.java} (78%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{DocValuesIteratorFactory.java => DocValuesIteratorAdapter.java} (85%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{StarTreeDocValuesIteratorFactory.java => StarTreeDocValuesIteratorAdapter.java} (88%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{MultipleTreesBuilder.java => StarTreesBuilder.java} (91%) rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/{BaseSingleStarTreeBuilderTests.java => BaseStarTreeBuilderTests.java} (93%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java similarity index 78% rename from server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java rename to server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 7b0957b1328de..f2f1f35d7c23f 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -23,7 +23,7 @@ import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; -import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -44,12 +44,9 @@ /** * Base class for star-tree builder */ -public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { +public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { - // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined - public static final String STAR_TREE_CODEC = "startreecodec"; - - private static final Logger logger = LogManager.getLogger(BaseSingleStarTreeBuilder.class); + private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); public static final int STAR_IN_DOC_VALUES_INDEX = -1; @@ -59,15 +56,13 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { protected final int numMetrics; protected final int numDimensions; - protected int numDocs; - protected int totalDocs; - protected int numNodes; + protected int numStarTreeDocs; + protected int totalSegmentDocs; + protected int numStarTreeNodes; protected final int maxLeafDocuments; protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); - // TODO: This will be initialized with OnHeap / OffHeap Implementations (Commented it's occurrences for now) - // private IndexOutput indexOutput; protected DocIdSetIterator[] dimensionReaders; protected DocIdSetIterator[] metricReaders; @@ -76,7 +71,7 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { protected DocValuesConsumer docValuesConsumer; protected DocValuesProducer docValuesProducer; - private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; + private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; private final StarTreeFieldConfiguration starTreeFieldSpec; private final List metricStatFieldPairs; @@ -91,7 +86,7 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { * @param state stores the segment state * @param mapperService helps to find the original type of the field */ - protected BaseSingleStarTreeBuilder( + protected BaseStarTreeBuilder( StarTreeField starTreeField, DocValuesProducer docValuesProducer, DocValuesConsumer docValuesConsumer, @@ -99,26 +94,21 @@ protected BaseSingleStarTreeBuilder( MapperService mapperService ) throws IOException { - logger.info("Building in base star tree builder"); - - // String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); - // logger.info("Star tree file name : {}", docFileName); + logger.debug("Building in base star tree builder"); - // indexOutput = state.directory.createOutput(docFileName, state.context); - // CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); this.mapperService = mapperService; this.starTreeField = starTreeField; this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); this.docValuesConsumer = docValuesConsumer; this.docValuesProducer = docValuesProducer; - this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); + this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); this.numDimensions = dimensionsSplitOrder.size(); this.dimensionsSplitOrder = new String[numDimensions]; this.skipStarNodeCreationForDimensions = new HashSet<>(); - this.totalDocs = state.segmentInfo.maxDoc(); + this.totalSegmentDocs = state.segmentInfo.maxDoc(); this.dimensionReaders = new DocIdSetIterator[numDimensions]; Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); @@ -130,7 +120,7 @@ protected BaseSingleStarTreeBuilder( } FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); - dimensionReaders[i] = starTreeDocValuesIteratorFactory.createIterator( + dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, docValuesProducer @@ -160,7 +150,7 @@ protected BaseSingleStarTreeBuilder( String metricName = metricStatFieldPair.getField(); FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); - metricReaders[index] = starTreeDocValuesIteratorFactory.createIterator( + metricReaders[index] = starTreeDocValuesIteratorFactory.getDocValuesIterator( metricDocValuesType, metricFieldInfos, docValuesProducer @@ -237,9 +227,9 @@ public List generateMetricStatFieldPairs() { *

* * @param startDocId Start document id in the star-tree @@ -247,15 +237,15 @@ public List generateMetricStatFieldPairs() { * @param dimensionId Dimension id of the star-node * @return Iterator for the aggregated starTreeDocument */ - public abstract Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + public abstract Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException; /** - * Returns the next segment star-tree document + * Returns the segment star-tree document */ - protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { - long[] dimensions = getNextSegmentStarTreeDocumentDimensions(); - Object[] metrics = getNextSegmentStarTreeDocumentMetrics(); + protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { + long[] dimensions = getStarTreeDimensionsFromSegment(); + Object[] metrics = getStarTreeMetricsFromSegment(); return new StarTreeDocument(dimensions, metrics); } @@ -265,7 +255,7 @@ protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { * @return dimension values for each of the star-tree dimension * @throws IOException when we are unable to iterate to the next doc */ - long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { + long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; for (int i = 0; i < numDimensions; i++) { try { @@ -292,7 +282,7 @@ long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { * @return metric values for each of the star-tree metric * @throws IOException when we are unable to iterate to the next doc */ - private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { + private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { // Ignore the column for COUNT aggregation function @@ -314,26 +304,26 @@ private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { * *

Will create a new aggregated star-tree document if the current one is {@code null}. * - * @param aggregatedStarTreeDocument Aggregated star-tree document - * @param segmentStarTreeDocument Segment star-tree document + * @param aggregatedSegmentDocument Aggregated star-tree document + * @param segmentDocument Segment star-tree document * @return Merged starTreeDocument */ - protected StarTreeDocument aggregateSegmentStarTreeDocument( - StarTreeDocument aggregatedStarTreeDocument, - StarTreeDocument segmentStarTreeDocument + protected StarTreeDocument aggregateSegmentDocuments( + StarTreeDocument aggregatedSegmentDocument, + StarTreeDocument segmentDocument ) { // TODO: HANDLE KEYWORDS LATER! - if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(segmentStarTreeDocument.dimensions, numDimensions); + if (aggregatedSegmentDocument == null) { + long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentStarTreeDocument.metrics[i], numericType); + metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentDocument.metrics[i], numericType); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot parse initial aggregated value", e); throw new IllegalArgumentException( - "Cannot parse initial aggregated value [" + segmentStarTreeDocument.metrics[i] + "]" + "Cannot parse initial aggregated value [" + segmentDocument.metrics[i] + "]" ); } } @@ -342,17 +332,17 @@ protected StarTreeDocument aggregateSegmentStarTreeDocument( for (int i = 0; i < numMetrics; i++) { try { StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( - aggregatedStarTreeDocument.metrics[i], - (Long) segmentStarTreeDocument.metrics[i], + aggregatedSegmentDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + aggregatedSegmentDocument.metrics[i], + (Long) segmentDocument.metrics[i], numericType ); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot apply segment raw value", e); - throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentDocument.metrics[i] + "]"); } } - return aggregatedStarTreeDocument; + return aggregatedSegmentDocument; } } @@ -361,54 +351,51 @@ protected StarTreeDocument aggregateSegmentStarTreeDocument( * *

Will create a new aggregated starTreeDocument if the current one is {@code null}. * - * @param aggregatedStarTreeDocument Aggregated star-tree document - * @param starTreeStarTreeDocument Star-tree document + * @param aggregatedDocument Aggregated star-tree document + * @param starTreeDocument Star-tree document * @return Merged star-tree document */ - public StarTreeDocument aggregateStarTreeDocument( - StarTreeDocument aggregatedStarTreeDocument, - StarTreeDocument starTreeStarTreeDocument + public StarTreeDocument aggregateDocuments( + StarTreeDocument aggregatedDocument, + StarTreeDocument starTreeDocument ) { // aggregate the documents - if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument.dimensions, numDimensions); + if (aggregatedDocument == null) { + long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = valueAggregators[i].cloneAggregatedValue(starTreeStarTreeDocument.metrics[i]); + metrics[i] = valueAggregators[i].getAggregatedValue(starTreeDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot clone aggregated value", e); - throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( - starTreeStarTreeDocument.metrics[i], - aggregatedStarTreeDocument.metrics[i] + aggregatedDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( + starTreeDocument.metrics[i], + aggregatedDocument.metrics[i] ); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot apply aggregated value", e); - throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeDocument.metrics[i] + "]"); } } - return aggregatedStarTreeDocument; + return aggregatedDocument; } } - // TODO: This will be taken care in off heap implementation for merges - // public abstract void build(List starTreeValues) throws IOException; - public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.info("Tree of Aggregations build is a go with config {}", starTreeField); + logger.debug("Tree of Aggregations build is a go with config {}", starTreeField); - Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalDocs); - logger.info("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalSegmentDocs); + logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); - logger.info("Finished Building TOA in ms : {}", (System.currentTimeMillis() - startTime)); + logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); } /** @@ -418,41 +405,36 @@ public void build() throws IOException { * @throws IOException when we are unable to build star-tree */ public void build(Iterator starTreeDocumentIterator) throws IOException { - int numSegmentStarTreeDocument = totalDocs; + int numSegmentStarTreeDocument = totalSegmentDocs; while (starTreeDocumentIterator.hasNext()) { appendToStarTree(starTreeDocumentIterator.next()); } - int numStarTreeDocument = numDocs; - logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + int numStarTreeDocument = numStarTreeDocs; + logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); - if (numDocs == 0) { + if (numStarTreeDocs == 0) { // TODO: Uncomment when segment codec is ready // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); return; } - constructStarTree(rootNode, 0, numDocs); - int numStarTreeDocumentUnderStarNode = numDocs - numStarTreeDocument; - logger.info( + constructStarTree(rootNode, 0, numStarTreeDocs); + int numStarTreeDocumentUnderStarNode = numStarTreeDocs - numStarTreeDocument; + logger.debug( "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", - numNodes, + numStarTreeNodes, numStarTreeDocumentUnderStarNode ); createAggregatedDocs(rootNode); - int numAggregatedStarTreeDocument = numDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; - logger.info("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + // TODO: When StarTree Codec is ready // Create doc values indices in disk - // TODO: Uncomment when segment codec is ready - // createSortedDocValuesIndices(docValuesConsumer); - // Serialize and save in disk - // TODO: Uncomment when segment codec is ready - // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); - - // TODO: Write star tree metadata for off heap implementation + // Write star tree metadata for off heap implementation } @@ -464,7 +446,7 @@ public void build(Iterator starTreeDocumentIterator) throws IO */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { appendStarTreeDocument(starTreeDocument); - numDocs++; + numStarTreeDocs++; } /** @@ -473,7 +455,7 @@ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOExcept * @return return new star-tree node */ private StarTreeBuilderUtils.TreeNode getNewNode() { - numNodes++; + numStarTreeNodes++; return new StarTreeBuilderUtils.TreeNode(); } @@ -560,12 +542,12 @@ private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endD StarTreeBuilderUtils.TreeNode starNode = getNewNode(); starNode.dimensionId = dimensionId; starNode.dimensionValue = StarTreeBuilderUtils.ALL; - starNode.startDocId = numDocs; - Iterator starTreeDocumentIterator = generateStarTreeForStarNode(startDocId, endDocId, dimensionId); + starNode.startDocId = numStarTreeDocs; + Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); while (starTreeDocumentIterator.hasNext()) { appendToStarTree(starTreeDocumentIterator.next()); } - starNode.endDocId = numDocs; + starNode.endDocId = numStarTreeDocs; return starNode; } @@ -588,13 +570,13 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If it has multiple documents, aggregate all of them for (int i = node.startDocId; i < node.endDocId; i++) { - aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, getStarTreeDocument(i)); + aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numDocs; + node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); } } else { @@ -612,13 +594,13 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If no star child exists, aggregate all aggregated documents from non-star children for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, createAggregatedDocs(child)); + aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numDocs; + node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); } } @@ -638,23 +620,7 @@ private long handleDateDimension(final String fieldName, final long val) { } public void close() throws IOException { - // boolean success = false; - // try { - // if (indexOutput != null) { - // indexOutput.writeInt(-1); - // CodecUtil.writeFooter(indexOutput); // write checksum - // } - // success = true; - // } catch (Exception e) { - // throw new RuntimeException(e); - // } finally { - // if (success) { - // IOUtils.close(indexOutput); - // } else { - // IOUtils.closeWhileHandlingException(indexOutput); - // } - // indexOutput = null; - // } + } } diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index b5a5a83ac3af9..71eafc182c573 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -202,13 +202,12 @@ public Void visit(Binding binding) { if (!overriddenKeys.remove(binding.getKey())) { super.visit(binding); - // record when a scope instance is used in a binding + // Record when a scope instance is used in a binding Scope scope = getScopeInstanceOrNull(binding); if (scope != null) { scopeInstancesInUse.put(scope, binding.getSource()); } } - return null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index a3a14f9bbb11a..36caf48ef0fb7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -53,7 +53,7 @@ public Double applyAggregatedValue(Double value, Double aggregatedValue) { } @Override - public Double cloneAggregatedValue(Double value) { + public Double getAggregatedValue(Double value) { return value; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index b9fa3d5b69a3f..c139a18eef91b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -45,7 +45,7 @@ public interface ValueAggregator { /** * Clones an aggregated value. */ - A cloneAggregatedValue(A value); + A getAggregatedValue(A value); /** * Returns the maximum size in bytes of the aggregated values seen so far. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index ed41445ac5cdf..cd7ec84015848 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -38,8 +38,10 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.FLOAT; case LONG: return StarTreeNumericType.LONG; - default: + case DOUBLE: return StarTreeNumericType.DOUBLE; + default: + throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java similarity index 85% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java index 66302f84f1449..6dc54e7139e9f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java @@ -19,12 +19,12 @@ * An interface to support iterators for various doc values types. * @opensearch.experimental */ -public interface DocValuesIteratorFactory { +public interface DocValuesIteratorAdapter { /** * Creates an iterator for the given doc values type and field using the doc values producer */ - DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; + DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; /** * Returns the next value for the given iterator diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java index 6baaedb1d0710..26e217d91dbc3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -9,7 +9,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; @@ -24,7 +24,7 @@ /** * On heap single tree builder */ -public class OnHeapSingleTreeBuilder extends BaseSingleStarTreeBuilder { +public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); @@ -69,17 +69,11 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { return starTreeDocuments.get(docId).dimensions[dimensionId]; } - // Handles star-tree rebuilds during merges :) - // @Override - // public void build(List starTreeValues) throws IOException { - // TODO: This will be handled during off heap implementation for merges - // } - @Override public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { - starTreeDocuments[i] = getNextSegmentStarTreeDocument(); + starTreeDocuments[i] = getSegmentStarTreeDocument(); } return processStarTreeDocuments(starTreeDocuments); } @@ -96,7 +90,7 @@ public Iterator processStarTreeDocuments(StarTreeDocument[] st Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = 0; i < numDimensions; i++) { if (o1.dimensions[i] != o2.dimensions[i]) { - return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } return 0; @@ -125,14 +119,14 @@ public boolean hasNext() { @Override public StarTreeDocument next() { // aggregate as we move on to the next doc - StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); while (docId < starTreeDocuments.length) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateStarTreeDocument(next, starTreeDocument); + next = aggregateDocuments(next, starTreeDocument); } } hasNext = false; @@ -150,7 +144,7 @@ public StarTreeDocument next() { * @throws IOException throws when unable to generate star-tree for star-node */ @Override - public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { int numDocs = endDocId - startDocId; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { @@ -185,7 +179,7 @@ public boolean hasNext() { @Override public StarTreeDocument next() { - StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; while (docId < numDocs) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; @@ -193,7 +187,7 @@ public StarTreeDocument next() { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateStarTreeDocument(next, starTreeDocument); + next = aggregateDocuments(next, starTreeDocument); } } hasNext = false; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java index 3b26f4c5e04c3..548a5261a78a3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -22,12 +22,4 @@ public interface SingleTreeBuilder extends Closeable { * @throws IOException when we are unable to build star-tree */ void build() throws Exception; - - /** - * Builds the star tree using star-tree document values during segment merges - * @param starTreeValues star-tree document values - * @throws IOException when we are unable to build star-tree - */ - // void build(List starTreeValues) throws IOException; - } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java similarity index 88% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index 0105ff42045de..a48b944b4e6c7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -21,10 +21,10 @@ * A factory class to return respective doc values iterator based on the doc volues type. * @opensearch.experimental */ -public class StarTreeDocValuesIteratorFactory implements DocValuesIteratorFactory { +public class StarTreeDocValuesIteratorAdapter implements DocValuesIteratorAdapter { @Override - public DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { case SORTED_SET: return producer.getSortedSet(field); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java similarity index 91% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index ad3d33474840f..e8ec930c4a3a0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -22,9 +22,9 @@ import java.util.List; import java.util.Locale; -public class MultipleTreesBuilder implements Closeable { +public class StarTreesBuilder implements Closeable { - private static final Logger logger = LogManager.getLogger(MultipleTreesBuilder.class); + private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class); private final List starTreeFields; private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; @@ -33,7 +33,7 @@ public class MultipleTreesBuilder implements Closeable { private final DocValuesProducer docValuesProducer; private final MapperService mapperService; - public MultipleTreesBuilder( + public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, DocValuesProducer docValuesProducer, @@ -58,7 +58,7 @@ public MultipleTreesBuilder( public void build() throws Exception { long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); - logger.info("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + logger.debug("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); // Build all star-trees for (int i = 0; i < numStarTrees; i++) { @@ -76,7 +76,7 @@ public void build() throws Exception { singleTreeBuilder.build(); } } - logger.info( + logger.debug( "Took {} ms to building {} star-trees with configs: {} using {} builder", System.currentTimeMillis() - startTime, numStarTrees, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 9fa980277f27b..e1841b3c2ff68 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -7,19 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.store.IndexOutput; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.List; import java.util.Map; -import java.util.Queue; - -import static java.nio.charset.StandardCharsets.UTF_8; /** * Util class for building star tree @@ -27,18 +15,10 @@ */ public class StarTreeBuilderUtils { - private static final Logger logger = LogManager.getLogger(StarTreeBuilderUtils.class); - - // TODO: To be moved to off heap star tree implementation - public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; - public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; - public static final long SERIALIZABLE_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES - * NUM_LONG_SERIALIZABLE_FIELDS); - - private StarTreeBuilderUtils() {} + private StarTreeBuilderUtils() { + } public static final int ALL = -1; - public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; /** Tree node representation */ public static class TreeNode { @@ -51,81 +31,4 @@ public static class TreeNode { public Map children; } - /** Serializes the tree */ - public static void serializeTree(IndexOutput indexOutput, TreeNode rootNode, String[] dimensions, int numNodes) throws IOException { - int headerSizeInBytes = computeHeaderByteSize(dimensions); - long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_SIZE_IN_BYTES; - - logger.info("Star tree size in bytes : {}", totalSizeInBytes); - - writeHeader(indexOutput, headerSizeInBytes, dimensions, numNodes); - writeNodes(indexOutput, rootNode); - } - - /** Computes the size of the header for the tree */ - static int computeHeaderByteSize(String[] dimensions) { - // Magic marker (8), version (4), size of header (4) and number of dimensions (4) - int headerSizeInBytes = 20; - - for (String dimension : dimensions) { - headerSizeInBytes += Integer.BYTES; // For dimension index - headerSizeInBytes += Integer.BYTES; // For length of dimension name - headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name - } - - headerSizeInBytes += Integer.BYTES; // For number of nodes. - return headerSizeInBytes; - } - - /** Writes the header of the tree */ - static void writeHeader(IndexOutput output, int headerSizeInBytes, String[] dimensions, int numNodes) throws IOException { - output.writeLong(MAGIC_MARKER); - output.writeInt(1); - output.writeInt(headerSizeInBytes); - output.writeInt(dimensions.length); - for (int i = 0; i < dimensions.length; i++) { - output.writeInt(i); - output.writeString(dimensions[i]); - } - output.writeInt(numNodes); - } - - /** Writes the nodes of the tree */ - static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException { - Queue queue = new LinkedList<>(); - queue.add(rootNode); - - int currentNodeId = 0; - while (!queue.isEmpty()) { - TreeNode node = queue.remove(); - - if (node.children == null) { - writeNode(output, node, ALL, ALL); - } else { - // Sort all children nodes based on dimension value - List sortedChildren = new ArrayList<>(node.children.values()); - sortedChildren.sort(Comparator.comparingLong(o -> o.dimensionValue)); - - int firstChildId = currentNodeId + queue.size() + 1; - int lastChildId = firstChildId + sortedChildren.size() - 1; - writeNode(output, node, firstChildId, lastChildId); - - queue.addAll(sortedChildren); - } - - currentNodeId++; - } - } - - /** Writes a node of the tree */ - private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException { - output.writeInt(node.dimensionId); - output.writeLong(node.dimensionValue); - output.writeInt(node.startDocId); - output.writeInt(node.endDocId); - output.writeInt(node.aggregatedDocId); - output.writeInt(firstChildId); - output.writeInt(lastChildId); - } - } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 58709934c894e..f30a9c82b1fd3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -40,8 +40,8 @@ public void testApplyAggregatedValue() { assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); } - public void testCloneAggregatedValue() { - assertEquals(3.14, aggregator.cloneAggregatedValue(3.14), 0.0); + public void testGetAggregatedValue() { + assertEquals(3.14, aggregator.getAggregatedValue(3.14), 0.0); } public void testGetMaxAggregatedValueByteSize() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java similarity index 93% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 82be99faa8bd8..753297a8004ce 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -52,9 +52,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class BaseSingleStarTreeBuilderTests extends OpenSearchTestCase { +public class BaseStarTreeBuilderTests extends OpenSearchTestCase { - private static BaseSingleStarTreeBuilder builder; + private static BaseStarTreeBuilder builder; private static MapperService mapperService; private static List dimensionsOrder; private static List fields = List.of( @@ -153,7 +153,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseSingleStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -178,7 +178,7 @@ public Iterator processSegmentStarTreeDocuments(int numDocs) t } @Override - public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { return null; } @@ -194,12 +194,12 @@ public void test_generateMetricStatFieldPairs() throws IOException { assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); } - public void test_aggregateStarTreeDocument() { + public void test_aggregateDocuments() { StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); - StarTreeDocument mergedStarTreeDocument = builder.aggregateStarTreeDocument(starTreeDocument1, starTreeDocument2); + StarTreeDocument mergedStarTreeDocument = builder.aggregateDocuments(starTreeDocument1, starTreeDocument2); assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index 7f6978dbf2720..f64da30eb591d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -29,12 +29,12 @@ public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase { - private static StarTreeDocValuesIteratorFactory factory; + private static StarTreeDocValuesIteratorAdapter factory; private static FieldInfo mockFieldInfo; @BeforeClass public static void setup() { - factory = new StarTreeDocValuesIteratorFactory(); + factory = new StarTreeDocValuesIteratorAdapter(); mockFieldInfo = new FieldInfo( "field", 1, @@ -60,7 +60,7 @@ public void testCreateIterator_SortedSet() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); + DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); assertEquals(iterator.getClass(), result.getClass()); } @@ -68,14 +68,14 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); + DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); assertEquals(iterator.getClass(), result.getClass()); } public void testCreateIterator_UnsupportedType() { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { - factory.createIterator(DocValuesType.BINARY, mockFieldInfo, producer); + factory.getDocValuesIterator(DocValuesType.BINARY, mockFieldInfo, producer); }); assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } From a90a55ef2e9ee5ddc5a01e70510a113321c078cf Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 25 Jun 2024 20:57:18 +0530 Subject: [PATCH 03/16] addressed major nits Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 314 ++++++++------- .../lucene/index/StarTreeDocValuesWriter.java | 37 -- .../compositeindex/datacube/MetricStat.java | 3 +- .../startree/{data => }/StarTreeDocument.java | 6 +- .../aggregators/CountValueAggregator.java | 74 ++++ .../MetricAggregationDescriptor.java | 144 +++++++ .../aggregators/MetricStatFieldPair.java | 115 ------ .../aggregators/SumValueAggregator.java | 22 +- .../startree/aggregators/ValueAggregator.java | 14 +- .../aggregators/ValueAggregatorFactory.java | 11 +- .../numerictype/StarTreeNumericType.java | 4 + .../StarTreeNumericTypeConverters.java | 6 + .../builder/DocValuesIteratorAdapter.java | 38 -- .../builder/OnHeapSingleTreeBuilder.java | 33 +- .../startree/builder/SingleTreeBuilder.java | 3 + .../StarTreeDocValuesIteratorAdapter.java | 19 +- .../startree/builder/StarTreesBuilder.java | 20 +- .../datacube/startree/data/DataType.java | 67 ---- .../datacube/startree/node/package-info.java | 13 - .../startree/utils/StarTreeBuilderUtils.java | 9 +- .../MetricAggregationDescriptorTests.java | 110 +++++ .../aggregators/MetricStatFieldPairTests.java | 69 ---- .../aggregators/SumValueAggregatorTests.java | 17 +- .../ValueAggregatorFactoryTests.java | 24 +- .../builder/BaseStarTreeBuilderTests.java | 37 +- .../builder/OnHeapSingleTreeBuilderTests.java | 378 ++++++++++++++++-- .../StarTreeValuesIteratorFactoryTests.java | 6 +- 27 files changed, 958 insertions(+), 635 deletions(-) delete mode 100644 server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/{data => }/StarTreeDocument.java (79%) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java delete mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index f2f1f35d7c23f..425101678ba58 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -16,15 +16,14 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -43,6 +42,7 @@ /** * Base class for star-tree builder + * @opensearch.experimental */ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { @@ -50,10 +50,9 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { public static final int STAR_IN_DOC_VALUES_INDEX = -1; - protected final String[] dimensionsSplitOrder; protected final Set skipStarNodeCreationForDimensions; - protected final String[] metrics; + protected final List metricAggregationDescriptors; protected final int numMetrics; protected final int numDimensions; protected int numStarTreeDocs; @@ -64,31 +63,25 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); protected DocIdSetIterator[] dimensionReaders; - protected DocIdSetIterator[] metricReaders; - protected ValueAggregator[] valueAggregators; - protected IndexNumericFieldData.NumericType[] numericTypes; + protected Map fieldProducerMap; protected DocValuesConsumer docValuesConsumer; - protected DocValuesProducer docValuesProducer; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; - private final StarTreeFieldConfiguration starTreeFieldSpec; - private final List metricStatFieldPairs; - private final MapperService mapperService; /** * Constructor for base star-tree builder * * @param starTreeField holds the configuration for the star tree - * @param docValuesProducer helps return the doc values iterator for each type based on field name + * @param fieldProducerMap helps return the doc values iterator for each type based on field name * @param docValuesConsumer to consume the new aggregated metrics during flush - * @param state stores the segment state + * @param state stores the segment write state * @param mapperService helps to find the original type of the field */ protected BaseStarTreeBuilder( StarTreeField starTreeField, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService @@ -96,25 +89,22 @@ protected BaseStarTreeBuilder( logger.debug("Building in base star tree builder"); - this.mapperService = mapperService; this.starTreeField = starTreeField; - this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); + StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); this.docValuesConsumer = docValuesConsumer; - this.docValuesProducer = docValuesProducer; + this.fieldProducerMap = fieldProducerMap; this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); this.numDimensions = dimensionsSplitOrder.size(); - this.dimensionsSplitOrder = new String[numDimensions]; this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = state.segmentInfo.maxDoc(); this.dimensionReaders = new DocIdSetIterator[numDimensions]; - Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); + Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); for (int i = 0; i < numDimensions; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - this.dimensionsSplitOrder[i] = dimension; if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { this.skipStarNodeCreationForDimensions.add(i); } @@ -123,41 +113,12 @@ protected BaseStarTreeBuilder( dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, - docValuesProducer + fieldProducerMap.get(dimensionFieldInfos.name) ); } - this.metricStatFieldPairs = generateMetricStatFieldPairs(); - this.numMetrics = metricStatFieldPairs.size(); - this.metrics = new String[numMetrics]; - this.valueAggregators = new ValueAggregator[numMetrics]; - this.numericTypes = new IndexNumericFieldData.NumericType[numMetrics]; - this.metricReaders = new DocIdSetIterator[numMetrics]; - - int index = 0; - for (MetricStatFieldPair metricStatFieldPair : metricStatFieldPairs) { - metrics[index] = metricStatFieldPair.toFieldName(); - valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricStatFieldPair.getMetricStat()); - - Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metrics[index]); - if (fieldMapper instanceof NumberFieldMapper) { - numericTypes[index] = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); - } else { - numericTypes[index] = IndexNumericFieldData.NumericType.DOUBLE; - } - // Ignore the column for COUNT aggregation function - if (valueAggregators[index].getAggregationType() != MetricStat.COUNT) { - String metricName = metricStatFieldPair.getField(); - FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); - DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); - metricReaders[index] = starTreeDocValuesIteratorFactory.getDocValuesIterator( - metricDocValuesType, - metricFieldInfos, - docValuesProducer - ); - } - index++; - } + this.metricAggregationDescriptors = generateMetricStatFieldPairs(mapperService, state); + this.numMetrics = metricAggregationDescriptors.size(); this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); } @@ -166,82 +127,98 @@ protected BaseStarTreeBuilder( * * @return list of metric stat mapped with respective fields */ - public List generateMetricStatFieldPairs() { - List metricStatFieldPairs = new ArrayList<>(); + public List generateMetricStatFieldPairs(MapperService mapperService, SegmentWriteState state) + throws IOException { + List metricAggregationDescriptors = new ArrayList<>(); + IndexNumericFieldData.NumericType numericType; + DocIdSetIterator metricStatReader = null; for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { - MetricStatFieldPair metricStatFieldPair = new MetricStatFieldPair(metricType, metric.getField()); - metricStatFieldPairs.add(metricStatFieldPair); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); + if (fieldMapper instanceof NumberFieldMapper) { + numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + } else { + logger.error("metric mapper is not of type number field mapper"); + throw new IllegalStateException("metric mapper is not of type number field mapper"); + } + // Ignore the column for COUNT aggregation function + if (metricType != MetricStat.COUNT) { + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); + DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); + metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( + metricDocValuesType, + metricFieldInfos, + fieldProducerMap.get(metricFieldInfos.name) + ); + } + MetricAggregationDescriptor metricAggregationDescriptor = new MetricAggregationDescriptor( + metricType, + metric.getField(), + numericType, + metricStatReader + ); + metricAggregationDescriptors.add(metricAggregationDescriptor); } } - return metricStatFieldPairs; + return metricAggregationDescriptors; } /** - * Appends a star-tree document to the star-tree. + * Adds a document to the star-tree. * - * @param starTreeDocument star tree document to be appended + * @param starTreeDocument star tree document to be added + * @throws IOException if an I/O error occurs while adding the document */ public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException; /** - * Returns the star-tree document of the given document id in the star-tree. + * Returns the document of the given document id in the star-tree. * - * @param docId Document dd - * @return Star tree document + * @param docId document id + * @return star tree document + * @throws IOException if an I/O error occurs while fetching the star-tree document */ public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException; /** - * Returns the star-tree document of the given document id in the star-tree. + * Retrieves the list of star-tree documents in the star-tree. * - * @return Star tree document + * @return Star tree documents */ public abstract List getStarTreeDocuments() throws IOException; /** - * Returns the dimension value of the given document and dimension id in the star-tree. + * Returns the value of the dimension for the given dimension id and document in the star-tree. * - * @param docId Document Id - * @param dimensionId Dimension Id - * @return Dimension value + * @param docId document id + * @param dimensionId dimension id + * @return dimension value */ public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; /** - * Sorts and aggregates the star-tree Document in the segment, and returns a star-tree document iterator for all the + * Sorts and aggregates the star-tree document in the segment, and returns a star-tree document iterator for all the * aggregated star-tree document. * - *

This method reads star-tree document from segment and generates the initial star-tree document for the star-tree. - * - * @param numDocs Number of documents in the segment + * @param numDocs number of documents in the given segment * @return Iterator for the aggregated star-tree document */ - public abstract Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException; + public abstract Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException; /** * Generates aggregated star-tree document for star-node. * - *

This method will do the following steps: - * - *

- * - * @param startDocId Start document id in the star-tree - * @param endDocId End document id (exclusive) in the star-tree - * @param dimensionId Dimension id of the star-node - * @return Iterator for the aggregated starTreeDocument + * @param startDocId start document id (inclusive) in the star-tree + * @param endDocId end document id (exclusive) in the star-tree + * @param dimensionId dimension id of the star-node + * @return Iterator for the aggregated star-tree documents */ public abstract Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException; /** - * Returns the segment star-tree document + * Returns the star-tree document from the segment + * @throws IOException when we are unable to build a star tree document from the segment */ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { long[] dimensions = getStarTreeDimensionsFromSegment(); @@ -250,10 +227,10 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { } /** - * Returns the next segment star-tree document for the dimensions + * Returns the dimension values for the next document from the segment * * @return dimension values for each of the star-tree dimension - * @throws IOException when we are unable to iterate to the next doc + * @throws IOException when we are unable to iterate to the next doc for the given dimension readers */ long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; @@ -262,84 +239,100 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { dimensionReaders[i].nextDoc(); } catch (IOException e) { logger.error("unable to iterate to next doc", e); + } catch (NullPointerException e) { + logger.error("dimension does not have an associated reader", e); + throw new IllegalStateException("dimension should have a valid associated reader", e); + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); } if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { dimensions[i] = handleDateDimension( - dimensionsSplitOrder[i], - starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]) + starTreeField.getDimensionsOrder().get(i).getField(), + starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]) ); } else { - dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]); } } return dimensions; } /** - * Returns the next segment star-tree document for the metrics + * Returns the metric values for the next document from the segment * * @return metric values for each of the star-tree metric - * @throws IOException when we are unable to iterate to the next doc + * @throws IOException when we are unable to iterate to the next doc for the given metric readers */ private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { // Ignore the column for COUNT aggregation function - if (metricReaders[i] != null) { + DocIdSetIterator metricStatReader = metricAggregationDescriptors.get(i).getMetricStatReader(); + if (metricStatReader != null) { try { - metricReaders[i].nextDoc(); + metricStatReader.nextDoc(); } catch (IOException e) { - // TODO : handle null values in columns logger.error("unable to iterate to next doc", e); + } catch (NullPointerException e) { + logger.error("metric does not have an associated reader", e); + throw new IllegalStateException("metric should have a valid associated reader", e); + } catch (Exception e) { + logger.error("unable to read the metric values from the segment", e); + throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricReaders[i]); + metrics[i] = starTreeDocValuesIteratorFactory.getNextOrd(metricStatReader); } } return metrics; } /** - * Merges a segment star-tree document (raw) into the aggregated star-tree document. + * Merges a star-tree document from the segment into an aggregated star-tree document. + * A new aggregated star-tree document is created if the aggregated segment document is null. * - *

Will create a new aggregated star-tree document if the current one is {@code null}. - * - * @param aggregatedSegmentDocument Aggregated star-tree document - * @param segmentDocument Segment star-tree document - * @return Merged starTreeDocument + * @param aggregatedSegmentDocument aggregated star-tree document + * @param segmentDocument segment star-tree document + * @return merged star-tree document */ - protected StarTreeDocument aggregateSegmentDocuments( + protected StarTreeDocument reduceSegmentStarTreeDocuments( StarTreeDocument aggregatedSegmentDocument, StarTreeDocument segmentDocument ) { - // TODO: HANDLE KEYWORDS LATER! if (aggregatedSegmentDocument == null) { long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentDocument.metrics[i], numericType); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot parse initial aggregated value", e); - throw new IllegalArgumentException( - "Cannot parse initial aggregated value [" + segmentDocument.metrics[i] + "]" - ); + ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); + metrics[i] = metricValueAggregator.getInitialAggregatedValue((Long) segmentDocument.metrics[i], starTreeNumericType); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + logger.error("Cannot parse initial segment doc value", e); + throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot parse initial segment doc value", e); + throw new RuntimeException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - aggregatedSegmentDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); + aggregatedSegmentDocument.metrics[i] = metricValueAggregator.applySegmentRawValue( aggregatedSegmentDocument.metrics[i], (Long) segmentDocument.metrics[i], - numericType + starTreeNumericType ); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply segment raw value", e); - throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentDocument.metrics[i] + "]"); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + logger.error("Cannot apply segment doc value for aggregation", e); + throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot apply segment doc value for aggregation", e); + throw new RuntimeException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } } return aggregatedSegmentDocument; @@ -347,52 +340,62 @@ protected StarTreeDocument aggregateSegmentDocuments( } /** - * Merges a star-tree document (aggregated) into the aggregated document. - * - *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * Merges a star-tree document into an aggregated star-tree document. + * A new aggregated star-tree document is created if the aggregated document is null. * - * @param aggregatedDocument Aggregated star-tree document - * @param starTreeDocument Star-tree document - * @return Merged star-tree document + * @param aggregatedDocument aggregated star-tree document + * @param starTreeDocument segment star-tree document + * @return merged star-tree document */ - public StarTreeDocument aggregateDocuments( - StarTreeDocument aggregatedDocument, - StarTreeDocument starTreeDocument - ) { + public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocument, StarTreeDocument starTreeDocument) { // aggregate the documents if (aggregatedDocument == null) { long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = valueAggregators[i].getAggregatedValue(starTreeDocument.metrics[i]); + metrics[i] = metricAggregationDescriptors.get(i).getValueAggregators().getAggregatedValue(starTreeDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot clone aggregated value", e); - throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeDocument.metrics[i] + "]"); + logger.error("Cannot get aggregated value", e); + throw new IllegalArgumentException("Cannot get aggregated value[" + starTreeDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot get value for aggregation", e); + throw new RuntimeException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( - starTreeDocument.metrics[i], - aggregatedDocument.metrics[i] - ); + aggregatedDocument.metrics[i] = metricAggregationDescriptors.get(i) + .getValueAggregators() + .applyAggregatedValue(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply aggregated value", e); - throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeDocument.metrics[i] + "]"); + logger.error("Cannot apply value to aggregated document for aggregation", e); + throw new IllegalArgumentException( + "Cannot apply value to aggregated document for aggregation[" + starTreeDocument.metrics[i] + "]" + ); + } catch (Exception e) { + logger.error("Cannot apply value to aggregated document for aggregation", e); + throw new RuntimeException( + "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]" + ); } } return aggregatedDocument; } } + /** + * Builds the star tree using total segment documents + * + * @throws IOException when we are unable to build star-tree + */ public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.debug("Tree of Aggregations build is a go with config {}", starTreeField); + logger.debug("Star-tree build is a go with star tree field{}", starTreeField); - Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalSegmentDocs); + Iterator starTreeDocumentIterator = sortMergeAndAggregateStarTreeDocument(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -439,10 +442,10 @@ public void build(Iterator starTreeDocumentIterator) throws IO } /** - * Appends a starTreeDocument to star tree + * Adds a document to star-tree * * @param starTreeDocument star-tree document - * @throws IOException throws an exception if we are unable to append the doc + * @throws IOException throws an exception if we are unable to add the doc */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { appendStarTreeDocument(starTreeDocument); @@ -450,7 +453,7 @@ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOExcept } /** - * Returns a new node + * Returns a new star-tree node * * @return return new star-tree node */ @@ -460,11 +463,11 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { } /** - * Implements the algorithm to construct a star-tree based on star-tree documents + * Implements the algorithm to construct a star-tree * - * @param node star-tree node - * @param startDocId start document id - * @param endDocId end document id + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -542,6 +545,7 @@ private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endD StarTreeBuilderUtils.TreeNode starNode = getNewNode(); starNode.dimensionId = dimensionId; starNode.dimensionValue = StarTreeBuilderUtils.ALL; + starNode.isStarNode = true; starNode.startDocId = numStarTreeDocs; Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); while (starTreeDocumentIterator.hasNext()) { @@ -570,9 +574,12 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If it has multiple documents, aggregate all of them for (int i = node.startDocId; i < node.endDocId; i++) { - aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } assert aggregatedStarTreeDocument != null; + if (null == aggregatedStarTreeDocument) { + throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); + } for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } @@ -584,7 +591,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node if (node.children.containsKey((long) StarTreeBuilderUtils.ALL)) { // If it has star child, use the star child aggregated document directly for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - if (child.dimensionValue == StarTreeBuilderUtils.ALL) { + if (child.isStarNode) { aggregatedStarTreeDocument = createAggregatedDocs(child); node.aggregatedDocId = child.aggregatedDocId; } else { @@ -594,9 +601,12 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If no star child exists, aggregate all aggregated documents from non-star children for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } assert aggregatedStarTreeDocument != null; + if (null == aggregatedStarTreeDocument) { + throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); + } for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } diff --git a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java deleted file mode 100644 index e8d96226da2ee..0000000000000 --- a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.apache.lucene.index; - -/** - * A wrapper for {@link DocValuesWriter} that contains the {@link DocValuesType} of the doc - */ -public class StarTreeDocValuesWriter { - - private final DocValuesType docValuesType; - private final DocValuesWriter docValuesWriter; - - public StarTreeDocValuesWriter(DocValuesType docValuesType, DocValuesWriter docValuesWriter) { - this.docValuesType = docValuesType; - this.docValuesWriter = docValuesWriter; - } - - /** - * Get the doc values type - */ - public DocValuesType getDocValuesType() { - return docValuesType; - } - - /** - * Get the doc values writer - */ - public DocValuesWriter getDocValuesWriter() { - return docValuesWriter; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java index a5a6d07ad9bbc..fbde296b15f7e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -21,8 +21,7 @@ public enum MetricStat { AVG("avg"), SUM("sum"), MIN("min"), - MAX("max"), - UNSUPPORTED("unsupported"); + MAX("max"); private final String typeName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java similarity index 79% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java index cb4253e21c141..3d537be1c3227 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java @@ -6,13 +6,17 @@ * compatible open source license. */ -package org.opensearch.index.compositeindex.datacube.startree.data; +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.annotation.ExperimentalApi; import java.util.Arrays; /** * Star tree document + * @opensearch.experimental */ +@ExperimentalApi public class StarTreeDocument { public final long[] dimensions; public final Object[] metrics; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java new file mode 100644 index 0000000000000..b4837809b9706 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; + +/** + * Count value aggregator for star tree + * + * @opensearch.experimental + */ +public class CountValueAggregator implements ValueAggregator { + public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + + @Override + public MetricStat getAggregationType() { + return MetricStat.COUNT; + } + + @Override + public StarTreeNumericType getStarTreeNumericType() { + return STAR_TREE_NUMERIC_TYPE; + } + + @Override + public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return 1.0; + } + + @Override + public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return value + 1; + } + + @Override + public Double applyAggregatedValue(Double value, Double aggregatedValue) { + return value + aggregatedValue; + } + + @Override + public Double getAggregatedValue(Double value) { + return value; + } + + @Override + public int getMaxAggregatedValueByteSize() { + return Long.BYTES; + } + + @Override + public Long toLongValue(Double value) { + try { + return NumericUtils.doubleToSortableLong(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } + } + + @Override + public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { + try { + return type.getDoubleValue(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java new file mode 100644 index 0000000000000..f5e1055b6ef24 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java @@ -0,0 +1,144 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.fielddata.IndexNumericFieldData; + +import java.util.Comparator; + +/** + * Builds aggregation function and doc values field pair to support various aggregations + * @opensearch.experimental + */ +public class MetricAggregationDescriptor implements Comparable { + + public static final String DELIMITER = "__"; + public static final String STAR = "*"; + public static final MetricAggregationDescriptor COUNT_STAR = new MetricAggregationDescriptor( + MetricStat.COUNT, + STAR, + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + + private final String metricStatName; + private final MetricStat metricStat; + private final String field; + private final ValueAggregator valueAggregators; + private final StarTreeNumericType starTreeNumericType; + private final DocIdSetIterator metricStatReader; + + /** + * Constructor for MetricAggregationDescriptor + */ + public MetricAggregationDescriptor( + MetricStat metricStat, + String field, + IndexNumericFieldData.NumericType numericType, + DocIdSetIterator metricStatReader + ) { + this.metricStat = metricStat; + this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat); + this.starTreeNumericType = StarTreeNumericType.fromNumericType(numericType); + this.metricStatReader = metricStatReader; + if (metricStat == MetricStat.COUNT) { + this.field = STAR; + } else { + this.field = field; + } + this.metricStatName = toFieldName(); + } + + /** + * @return metric type + */ + public MetricStat getMetricStat() { + return metricStat; + } + + /** + * @return field Name + */ + public String getField() { + return field; + } + + /** + * @return the metric stat name + */ + public String getMetricStatName() { + return metricStatName; + } + + /** + * @return aggregator for the field value + */ + public ValueAggregator getValueAggregators() { + return valueAggregators; + } + + /** + * @return star tree numeric type + */ + public StarTreeNumericType getStarTreeNumericType() { + return starTreeNumericType; + } + + /** + * @return metric value reader iterator + */ + public DocIdSetIterator getMetricStatReader() { + return metricStatReader; + } + + /** + * @return field name with metric type and field + */ + public String toFieldName() { + return toFieldName(metricStat, field); + } + + /** + * Builds field name with metric type and field + */ + public static String toFieldName(MetricStat metricType, String field) { + return metricType.getTypeName() + DELIMITER + field; + } + + @Override + public int hashCode() { + return 31 * metricStat.hashCode() + field.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof MetricAggregationDescriptor) { + MetricAggregationDescriptor anotherPair = (MetricAggregationDescriptor) obj; + return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); + } + return false; + } + + @Override + public String toString() { + return toFieldName(); + } + + @Override + public int compareTo(MetricAggregationDescriptor other) { + return Comparator.comparing((MetricAggregationDescriptor o) -> o.field) + .thenComparing((MetricAggregationDescriptor o) -> o.metricStat) + .compare(this, other); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java deleted file mode 100644 index 3fb041373ff51..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.aggregators; - -import org.opensearch.index.compositeindex.datacube.MetricStat; - -import java.util.Comparator; - -/** - * Builds aggregation function and doc values field pair to support various aggregations - * @opensearch.experimental - */ -public class MetricStatFieldPair implements Comparable { - - public static final String DELIMITER = "__"; - public static final String STAR = "*"; - public static final MetricStatFieldPair COUNT_STAR = new MetricStatFieldPair(MetricStat.COUNT, STAR); - - private final MetricStat metricStat; - private final String field; - - /** - * Constructor for MetricStatFieldPair - */ - public MetricStatFieldPair(MetricStat metricStat, String field) { - this.metricStat = metricStat; - if (metricStat == MetricStat.COUNT) { - this.field = STAR; - } else { - this.field = field; - } - } - - /** - * @return Metric Type - */ - public MetricStat getMetricStat() { - return metricStat; - } - - /** - * @return field Name - */ - public String getField() { - return field; - } - - /** - * @return field name with metric type and field - */ - public String toFieldName() { - return toFieldName(metricStat, field); - } - - /** - * Builds field name with metric type and field - */ - public static String toFieldName(MetricStat metricType, String field) { - return metricType.getTypeName() + DELIMITER + field; - } - - /** - * Builds MetricStatFieldPair from field name - */ - public static MetricStatFieldPair fromFieldName(String fieldName) { - String[] parts = fieldName.split(DELIMITER, 2); - return fromMetricAndFieldName(parts[0], parts[1]); - } - - /** - * Builds MetricStatFieldPair from metric and field name - */ - private static MetricStatFieldPair fromMetricAndFieldName(String metricName, String fieldName) { - MetricStat metricType = MetricStat.fromTypeName(metricName); - if (metricType == MetricStat.COUNT) { - return COUNT_STAR; - } else { - return new MetricStatFieldPair(metricType, fieldName); - } - } - - @Override - public int hashCode() { - return 31 * metricStat.hashCode() + field.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj instanceof MetricStatFieldPair) { - MetricStatFieldPair anotherPair = (MetricStatFieldPair) obj; - return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); - } - return false; - } - - @Override - public String toString() { - return toFieldName(); - } - - @Override - public int compareTo(MetricStatFieldPair other) { - return Comparator.comparing((MetricStatFieldPair o) -> o.field) - .thenComparing((MetricStatFieldPair o) -> o.metricStat) - .compare(this, other); - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index 36caf48ef0fb7..d9e55caae8784 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -10,16 +10,16 @@ import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; import org.opensearch.search.aggregations.metrics.CompensatedSum; /** * Sum value aggregator for star tree * - * @opensearch.internal + * @opensearch.experimental */ public class SumValueAggregator implements ValueAggregator { - public static final DataType AGGREGATED_VALUE_TYPE = DataType.DOUBLE; + + public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -27,20 +27,20 @@ public MetricStat getAggregationType() { } @Override - public DataType getAggregatedValueType() { - return AGGREGATED_VALUE_TYPE; + public StarTreeNumericType getStarTreeNumericType() { + return STAR_TREE_NUMERIC_TYPE; } @Override - public Double getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType) { - return starTreeNumericType.getDoubleValue(rawValue); + public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return starTreeNumericType.getDoubleValue(segmentDocValue); } @Override - public Double applySegmentRawValue(Double value, Long rawValue, StarTreeNumericType starTreeNumericType) { + public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); - kahanSummation.add(starTreeNumericType.getDoubleValue(rawValue)); + kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); return kahanSummation.value(); } @@ -63,7 +63,7 @@ public int getMaxAggregatedValueByteSize() { } @Override - public Long convertAggregationTypeToSortableLongValue(Double value) { + public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { @@ -72,7 +72,7 @@ public Long convertAggregationTypeToSortableLongValue(Double value) { } @Override - public Double convertSortableLongToAggregatedTypeValue(Long value, StarTreeNumericType type) { + public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index c139a18eef91b..397f497d0105e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -9,10 +9,10 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. + * * @opensearch.experimental */ public interface ValueAggregator { @@ -25,17 +25,17 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - DataType getAggregatedValueType(); + StarTreeNumericType getStarTreeNumericType(); /** * Returns the initial aggregated value. */ - A getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType); + A getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** - * Applies a raw value to the current aggregated value. + * Applies a segment doc value to the current aggregated value. */ - A applySegmentRawValue(A value, Long rawValue, StarTreeNumericType starTreeNumericType); + A applySegmentRawValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies an aggregated value to the current aggregated value. @@ -55,10 +55,10 @@ public interface ValueAggregator { /** * Converts an aggregated value into a Long type. */ - Long convertAggregationTypeToSortableLongValue(A value); + Long toLongValue(A value); /** * Converts an aggregated value from a Long type. */ - A convertSortableLongToAggregatedTypeValue(Long rawValue, StarTreeNumericType type); + A toStarTreeNumericTypeValue(Long rawValue, StarTreeNumericType type); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 523b3fbec2e7c..2bde2f16f91f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -8,10 +8,11 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; /** * Value aggregator factory for a given aggregation type + * * @opensearch.experimental */ public class ValueAggregatorFactory { @@ -28,6 +29,8 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: return new SumValueAggregator(); + case COUNT: + return new CountValueAggregator(); default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } @@ -39,11 +42,13 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType) { * @param aggregationType Aggregation type * @return Data type of the aggregated value */ - public static DataType getAggregatedValueType(MetricStat aggregationType) { + public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationType) { switch (aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: - return SumValueAggregator.AGGREGATED_VALUE_TYPE; + return SumValueAggregator.STAR_TREE_NUMERIC_TYPE; + case COUNT: + return CountValueAggregator.STAR_TREE_NUMERIC_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index cd7ec84015848..28bfd82c69fdf 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -12,6 +12,10 @@ import java.util.function.Function; +/** + * Enum to map Star Tree Numeric Types to Lucene's Numeric Type + * @opensearch.experimental + */ public enum StarTreeNumericType { HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index 59a2c3418e931..f3fdd0d1162c3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -10,7 +10,13 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.annotation.ExperimentalApi; +/** + * Numeric converters used during aggregations of metric values + * @opensearch.experimental + */ +@ExperimentalApi public class StarTreeNumericTypeConverters { public static double halfFloatPointToDouble(Long value) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java deleted file mode 100644 index 6dc54e7139e9f..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.builder; - -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.search.DocIdSetIterator; - -import java.io.IOException; - -/** - * An interface to support iterators for various doc values types. - * @opensearch.experimental - */ -public interface DocValuesIteratorAdapter { - - /** - * Creates an iterator for the given doc values type and field using the doc values producer - */ - DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; - - /** - * Returns the next value for the given iterator - */ - long getNextValue(DocIdSetIterator iterator) throws IOException; - - /** - * Returns the doc id for the next document from the given iterator - */ - int nextDoc(DocIdSetIterator iterator) throws IOException; -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java index 26e217d91dbc3..0289a56ce5b5f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -11,8 +11,9 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -20,10 +21,13 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; /** * On heap single tree builder + * @opensearch.experimental */ +@ExperimentalApi public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); @@ -32,20 +36,20 @@ public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { * Constructor for OnHeapSingleTreeBuilder * * @param starTreeField star-tree field - * @param docValuesProducer document values producer + * @param fieldProducerMap helps with document values producer for a particular field * @param docValuesConsumer document values consumer * @param segmentWriteState segment write state - * @param mapperService + * @param mapperService helps with the numeric type of field * @throws IOException throws an exception we are unable to construct an onheap star-tree */ public OnHeapSingleTreeBuilder( StarTreeField starTreeField, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - super(starTreeField, docValuesProducer, docValuesConsumer, segmentWriteState, mapperService); + super(starTreeField, fieldProducerMap, docValuesConsumer, segmentWriteState, mapperService); } @Override @@ -70,12 +74,12 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { starTreeDocuments[i] = getSegmentStarTreeDocument(); } - return processStarTreeDocuments(starTreeDocuments); + return sortMergeAndAggregateStarTreeDocument(starTreeDocuments); } /** @@ -84,7 +88,7 @@ public Iterator processSegmentStarTreeDocuments(int numDocs) t * @return iterator for star-tree documents * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - public Iterator processStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(StarTreeDocument[] starTreeDocuments) throws IOException { // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { @@ -119,14 +123,14 @@ public boolean hasNext() { @Override public StarTreeDocument next() { // aggregate as we move on to the next doc - StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); + StarTreeDocument next = reduceSegmentStarTreeDocuments(null, currentStarTreeDocument); while (docId < starTreeDocuments.length) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateDocuments(next, starTreeDocument); + next = reduceSegmentStarTreeDocuments(next, starTreeDocument); } } hasNext = false; @@ -144,7 +148,8 @@ public StarTreeDocument next() { * @throws IOException throws when unable to generate star-tree for star-node */ @Override - public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException { int numDocs = endDocId - startDocId; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { @@ -153,7 +158,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = dimensionId + 1; i < numDimensions; i++) { if (o1.dimensions[i] != o2.dimensions[i]) { - return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } return 0; @@ -179,7 +184,7 @@ public boolean hasNext() { @Override public StarTreeDocument next() { - StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); + StarTreeDocument next = reduceStarTreeDocuments(null, currentStarTreeDocument); next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; while (docId < numDocs) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; @@ -187,7 +192,7 @@ public StarTreeDocument next() { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateDocuments(next, starTreeDocument); + next = reduceStarTreeDocuments(next, starTreeDocument); } } hasNext = false; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java index 548a5261a78a3..44866caba938b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -8,6 +8,8 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.opensearch.common.annotation.ExperimentalApi; + import java.io.Closeable; import java.io.IOException; @@ -15,6 +17,7 @@ * A star-tree builder that builds a single star-tree. * @opensearch.experimental */ +@ExperimentalApi public interface SingleTreeBuilder extends Closeable { /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index a48b944b4e6c7..e4cf2a45e131b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -14,16 +14,21 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; import java.io.IOException; /** * A factory class to return respective doc values iterator based on the doc volues type. + * * @opensearch.experimental */ -public class StarTreeDocValuesIteratorAdapter implements DocValuesIteratorAdapter { +@ExperimentalApi +public class StarTreeDocValuesIteratorAdapter { - @Override + /** + * Creates an iterator for the given doc values type and field using the doc values producer + */ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { case SORTED_SET: @@ -35,8 +40,10 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field } } - @Override - public long getNextValue(DocIdSetIterator iterator) throws IOException { + /** + * Returns the next ordinal for the given iterator + */ + public long getNextOrd(DocIdSetIterator iterator) throws IOException { if (iterator instanceof SortedSetDocValues) { return ((SortedSetDocValues) iterator).nextOrd(); } else if (iterator instanceof SortedNumericDocValues) { @@ -46,7 +53,9 @@ public long getNextValue(DocIdSetIterator iterator) throws IOException { } } - @Override + /** + * Returns the doc id for the next document from the given iterator + */ public int nextDoc(DocIdSetIterator iterator) throws IOException { return iterator.nextDoc(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index e8ec930c4a3a0..4d110609462a5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -13,6 +13,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.mapper.MapperService; @@ -21,7 +22,14 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Map; +/** + * Builder to construct star tree based on multiple star tree configs + * + * @opensearch.experimental + */ +@ExperimentalApi public class StarTreesBuilder implements Closeable { private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class); @@ -30,13 +38,13 @@ public class StarTreesBuilder implements Closeable { private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; private final DocValuesConsumer docValuesConsumer; private final SegmentWriteState state; - private final DocValuesProducer docValuesProducer; + private final Map fieldProducerMap; private final MapperService mapperService; public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService @@ -46,7 +54,7 @@ public StarTreesBuilder( throw new IllegalArgumentException("Must provide star-tree builder configs"); } this.buildMode = buildMode; - this.docValuesProducer = docValuesProducer; + this.fieldProducerMap = fieldProducerMap; this.docValuesConsumer = docValuesConsumer; this.state = segmentWriteState; this.mapperService = mapperService; @@ -67,7 +75,7 @@ public void build() throws Exception { SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( starTreeField, buildMode, - docValuesProducer, + fieldProducerMap, docValuesConsumer, state, mapperService @@ -93,14 +101,14 @@ public void close() throws IOException { private static SingleTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { switch (buildMode) { case ON_HEAP: - return new OnHeapSingleTreeBuilder(starTreeField, docValuesProducer, docValuesConsumer, state, mapperService); + return new OnHeapSingleTreeBuilder(starTreeField, fieldProducerMap, docValuesConsumer, state, mapperService); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java deleted file mode 100644 index 3c737c44b7219..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.data; - -/** - * Data type of doc values - * @opensearch.internal - */ -public enum DataType { - INT(Integer.BYTES, true), - LONG(Long.BYTES, true), - FLOAT(Float.BYTES, true), - DOUBLE(Double.BYTES, true); - - private final int size; - private final boolean numeric; - - DataType(int size, boolean numeric) { - this.size = size; - this.numeric = numeric; - } - - /** - * Returns the number of bytes needed to store the data type. - */ - public int size() { - if (size >= 0) { - return size; - } - throw new IllegalStateException("Cannot get number of bytes for: " + this); - } - - /** - * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), - * {@code false} otherwise. - */ - public boolean isNumeric() { - return numeric; - } - - /** - * Converts the given string value to the data type. Returns byte[] for BYTES. - */ - public Object convert(String value) { - try { - switch (this) { - case INT: - return Integer.valueOf(value); - case LONG: - return Long.valueOf(value); - case FLOAT: - return Float.valueOf(value); - case DOUBLE: - return Double.valueOf(value); - default: - throw new IllegalStateException(); - } - } catch (Exception e) { - throw new IllegalArgumentException(e); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java deleted file mode 100644 index 3af5020b76da2..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Node for Composite Index Star Tree - * @opensearch.experimental - */ -package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index e1841b3c2ff68..656196c7ffe0f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -7,27 +7,30 @@ */ package org.opensearch.index.compositeindex.datacube.startree.utils; +import org.opensearch.common.annotation.ExperimentalApi; + import java.util.Map; /** * Util class for building star tree * @opensearch.experimental */ +@ExperimentalApi public class StarTreeBuilderUtils { - private StarTreeBuilderUtils() { - } + private StarTreeBuilderUtils() {} public static final int ALL = -1; /** Tree node representation */ public static class TreeNode { public int dimensionId = ALL; - public long dimensionValue = ALL; public int startDocId = ALL; public int endDocId = ALL; public int aggregatedDocId = ALL; public int childDimensionId = ALL; + public long dimensionValue = ALL; + public boolean isStarNode = false; public Map children; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java new file mode 100644 index 0000000000000..98b79141df9eb --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.test.OpenSearchTestCase; + +public class MetricAggregationDescriptorTests extends OpenSearchTestCase { + + public void testConstructor() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(MetricStat.SUM, pair.getMetricStat()); + assertEquals("column1", pair.getField()); + } + + public void testCountStarConstructor() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.COUNT, + "anything", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + } + + public void testToFieldName() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.SUM, + "column2", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals("sum__column2", pair.toFieldName()); + } + + public void testEquals() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(pair1, pair2); + assertNotEquals( + pair1, + new MetricAggregationDescriptor(MetricStat.COUNT, "column1", IndexNumericFieldData.NumericType.DOUBLE, null) + ); + assertNotEquals(pair1, new MetricAggregationDescriptor(MetricStat.SUM, "column2", IndexNumericFieldData.NumericType.DOUBLE, null)); + } + + public void testHashCode() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(pair1.hashCode(), pair2.hashCode()); + } + + public void testCompareTo() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column2", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair3 = new MetricAggregationDescriptor( + MetricStat.COUNT, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertTrue(pair1.compareTo(pair2) < 0); + assertTrue(pair2.compareTo(pair1) > 0); + assertTrue(pair1.compareTo(pair3) > 0); + assertTrue(pair3.compareTo(pair1) < 0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java deleted file mode 100644 index fd026f49c0140..0000000000000 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators; - -import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.test.OpenSearchTestCase; - -public class MetricStatFieldPairTests extends OpenSearchTestCase { - - public void testConstructor() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(MetricStat.SUM, pair.getMetricStat()); - assertEquals("column1", pair.getField()); - } - - public void testCountStarConstructor() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.COUNT, "anything"); - assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); - } - - public void testToFieldName() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.AVG, "column2"); - assertEquals("avg__column2", pair.toFieldName()); - } - - public void testFromFieldName() { - MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("max__column3"); - assertEquals(MetricStat.MAX, pair.getMetricStat()); - assertEquals("column3", pair.getField()); - } - - public void testCountStarFromFieldName() { - MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("count__*"); - assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); - assertSame(MetricStatFieldPair.COUNT_STAR, pair); - } - - public void testEquals() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(pair1, pair2); - assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.AVG, "column1")); - assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.SUM, "column2")); - } - - public void testHashCode() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(pair1.hashCode(), pair2.hashCode()); - } - - public void testCompareTo() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column2"); - MetricStatFieldPair pair3 = new MetricStatFieldPair(MetricStat.AVG, "column1"); - assertTrue(pair1.compareTo(pair2) < 0); - assertTrue(pair2.compareTo(pair1) > 0); - assertTrue(pair1.compareTo(pair3) > 0); - assertTrue(pair3.compareTo(pair1) < 0); - } -} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index f30a9c82b1fd3..287deeead9d9d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -21,8 +21,8 @@ public void testGetAggregationType() { assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); } - public void testGetAggregatedValueType() { - assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, aggregator.getAggregatedValueType()); + public void testGetStarTreeNumericType() { + assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, aggregator.getStarTreeNumericType()); } public void testGetInitialAggregatedValue() { @@ -37,7 +37,6 @@ public void testApplySegmentRawValue() { public void testApplyAggregatedValue() { assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); - assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); } public void testGetAggregatedValue() { @@ -48,17 +47,13 @@ public void testGetMaxAggregatedValueByteSize() { assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); } - public void testConvertAggregationTypeToSortableLongValue() { + public void testToLongValue() { SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.convertAggregationTypeToSortableLongValue(3.14), 0.0); + assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.toLongValue(3.14), 0.0); } - public void testConvertSortableLongToAggregatedTypeValue() { + public void testToStarTreeNumericTypeValue() { SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals( - NumericUtils.sortableLongToDouble(3L), - aggregator.convertSortableLongToAggregatedTypeValue(3L, StarTreeNumericType.DOUBLE), - 0.0 - ); + assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index f80783dacb643..21e2d1304f085 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; public class ValueAggregatorFactoryTests extends OpenSearchTestCase { @@ -20,24 +20,8 @@ public void testGetValueAggregatorForSumType() { assertEquals(SumValueAggregator.class, aggregator.getClass()); } - public void testGetValueAggregatorForUnsupportedType() { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> ValueAggregatorFactory.getValueAggregator(MetricStat.UNSUPPORTED) - ); - assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); - } - - public void testGetAggregatedValueTypeForSumType() { - DataType dataType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); - assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, dataType); - } - - public void testGetAggregatedValueTypeForUnsupportedType() { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> ValueAggregatorFactory.getAggregatedValueType(MetricStat.UNSUPPORTED) - ); - assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + public void testGetStarTreeNumericTypeForSumType() { + StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); + assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, starTreeNumericType); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 753297a8004ce..7a13ab7701912 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -27,10 +27,11 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -46,6 +47,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -72,6 +74,7 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static List metrics; private static Directory directory; private static FieldInfo[] fieldsInfo; + private static SegmentWriteState state; @BeforeClass public static void setup() throws IOException { @@ -104,7 +107,7 @@ public static void setup() throws IOException { ); fieldsInfo = new FieldInfo[fields.size()]; - + Map fieldProducerMap = new HashMap<>(); for (int i = 0; i < fieldsInfo.length; i++) { fieldsInfo[i] = new FieldInfo( fields.get(i), @@ -125,16 +128,10 @@ public static void setup() throws IOException { false, false ); + fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - final SegmentWriteState state = new SegmentWriteState( - InfoStream.getDefault(), - segmentInfo.dir, - segmentInfo, - fieldInfos, - null, - newIOContext(random()) - ); + state = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -153,7 +150,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -173,7 +170,7 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { return null; } @@ -186,20 +183,20 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } public void test_generateMetricStatFieldPairs() throws IOException { - List metricStatFieldPairs = builder.generateMetricStatFieldPairs(); - List expectedMetricStatFieldPairs = List.of( - new MetricStatFieldPair(MetricStat.SUM, "field2"), - new MetricStatFieldPair(MetricStat.SUM, "field4") + List metricAggregationDescriptors = builder.generateMetricStatFieldPairs(mapperService, state); + List expectedMetricAggregationDescriptors = List.of( + new MetricAggregationDescriptor(MetricStat.SUM, "field2", IndexNumericFieldData.NumericType.DOUBLE, null), + new MetricAggregationDescriptor(MetricStat.SUM, "field4", IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); + assertEquals(metricAggregationDescriptors, expectedMetricAggregationDescriptors); } - public void test_aggregateDocuments() { + public void test_reduceStarTreeDocuments() { StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); - StarTreeDocument mergedStarTreeDocument = builder.aggregateDocuments(starTreeDocument1, starTreeDocument2); + StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(starTreeDocument1, starTreeDocument2); assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java index e81c836c9d723..73955080c36ae 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java @@ -19,16 +19,18 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -36,7 +38,7 @@ import org.opensearch.index.mapper.MappingLookup; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; -import org.junit.BeforeClass; +import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -44,6 +46,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -52,10 +55,10 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { - private static OnHeapSingleTreeBuilder builder; - private static MapperService mapperService; - private static List dimensionsOrder; - private static List fields = List.of( + private OnHeapSingleTreeBuilder builder; + private MapperService mapperService; + private List dimensionsOrder; + private List fields = List.of( "field1", "field2", "field3", @@ -67,19 +70,23 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { "field9", "field10" ); - private static List metrics; - private static Directory directory; - private static FieldInfo[] fieldsInfo; - - @BeforeClass - public static void setup() throws IOException { + private List metrics; + private Directory directory; + private FieldInfo[] fieldsInfo; + private StarTreeField compositeField; + private Map fieldProducerMap; + private DocValuesConsumer docValuesConsumer; + private SegmentWriteState writeState; + + @Before + public void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); - DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + docValuesConsumer = mock(DocValuesConsumer.class); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - StarTreeField compositeField = new StarTreeField( + compositeField = new StarTreeField( "test", dimensionsOrder, metrics, @@ -102,7 +109,7 @@ public static void setup() throws IOException { ); fieldsInfo = new FieldInfo[fields.size()]; - + fieldProducerMap = new HashMap<>(); for (int i = 0; i < fieldsInfo.length; i++) { fieldsInfo[i] = new FieldInfo( fields.get(i), @@ -123,16 +130,10 @@ public static void setup() throws IOException { false, false ); + fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - final SegmentWriteState writeState = new SegmentWriteState( - InfoStream.getDefault(), - segmentInfo.dir, - segmentInfo, - fieldInfos, - null, - newIOContext(random()) - ); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -150,10 +151,10 @@ public static void setup() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, writeState, mapperService); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); } - public void test_processStarTreeDocuments() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -169,10 +170,20 @@ public void test_processStarTreeDocuments() throws IOException { new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); int numOfAggregatedDocuments = 0; - while (starTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); @@ -189,7 +200,7 @@ public void test_processStarTreeDocuments() throws IOException { } - public void test_processStarTreeDocuments_nullDocument() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -200,9 +211,21 @@ public void test_processStarTreeDocuments_nullDocument() throws IOException { starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); - StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + Long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); @@ -210,27 +233,270 @@ public void test_processStarTreeDocuments_nullDocument() throws IOException { assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertThrows("Cannot apply aggregated value [null]", IllegalArgumentException.class, starTreeDocumentIterator::next); + assertThrows( + "Null metric should have resulted in IllegalStateException", + IllegalStateException.class, + segmentStarTreeDocumentIterator::next + ); } - public void test_build() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0 }); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0 }); starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE }); starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); - builder.build(starTreeDocumentIterator); + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_build_halfFloatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new long[] { 2, 4, 3, 4 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10) } + ); + starTreeDocuments[1] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6) } + ); + starTreeDocuments[2] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12) } + ); + starTreeDocuments[3] = new StarTreeDocument( + new long[] { 2, 4, 3, 4 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4) } + ); + starTreeDocuments[4] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16) } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[0]).numericValue().floatValue() + ); + long metric2 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size()); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + public void test_build_floatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + public void test_build_longMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = (Long) starTreeDocuments[i].metrics[0]; + long metric2 = (Long) starTreeDocuments[i].metrics[1]; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), @@ -241,7 +507,43 @@ public void test_build() throws IOException { new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) ); - Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator(); + return expectedStarTreeDocuments.iterator(); + } + + public void test_build() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + private static void assertStarTreeDocuments( + List resultStarTreeDocuments, + Iterator expectedStarTreeDocumentIterator + ) { Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index f64da30eb591d..f5d53afb8f774 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -84,7 +84,7 @@ public void testGetNextValue_SortedSet() throws IOException { SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); when(iterator.nextOrd()).thenReturn(42L); - long result = factory.getNextValue(iterator); + long result = factory.getNextOrd(iterator); assertEquals(42L, result); } @@ -92,14 +92,14 @@ public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextValue()).thenReturn(123L); - long result = factory.getNextValue(iterator); + long result = factory.getNextOrd(iterator); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextOrd(iterator); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); } From fdaf6ccf37e2ea2e724324687727ce3a24cf0093 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 27 Jun 2024 00:43:55 +0530 Subject: [PATCH 04/16] includes Count Aggregator Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 201 +++++++++--------- .../common/inject/util/Modules.java | 1 + .../aggregators/CountValueAggregator.java | 22 +- ...criptor.java => MetricAggregatorInfo.java} | 58 ++--- .../aggregators/SumValueAggregator.java | 37 ++-- .../startree/aggregators/ValueAggregator.java | 10 +- .../aggregators/ValueAggregatorFactory.java | 4 +- .../numerictype/StarTreeNumericType.java | 11 +- .../StarTreeNumericTypeConverters.java | 14 ++ ...uilder.java => OnHeapStarTreeBuilder.java} | 19 +- ...eTreeBuilder.java => StarTreeBuilder.java} | 4 +- .../StarTreeDocValuesIteratorAdapter.java | 14 +- .../startree/builder/StarTreesBuilder.java | 60 +++--- .../startree/utils/StarTreeBuilderUtils.java | 37 +++- .../CountValueAggregatorTests.java | 56 +++++ ...ts.java => MetricAggregatorInfoTests.java} | 43 ++-- .../aggregators/SumValueAggregatorTests.java | 46 ++-- .../ValueAggregatorFactoryTests.java | 4 +- .../builder/BaseStarTreeBuilderTests.java | 30 +-- ...s.java => OnHeapStarTreeBuilderTests.java} | 199 +++++++++-------- .../StarTreeValuesIteratorFactoryTests.java | 21 +- 21 files changed, 508 insertions(+), 383 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptor.java => MetricAggregatorInfo.java} (63%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilder.java => OnHeapStarTreeBuilder.java} (89%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{SingleTreeBuilder.java => StarTreeBuilder.java} (87%) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptorTests.java => MetricAggregatorInfoTests.java} (63%) rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilderTests.java => OnHeapStarTreeBuilderTests.java} (77%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 425101678ba58..af43dab7c498b 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -9,21 +9,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -41,10 +40,12 @@ import java.util.Set; /** - * Base class for star-tree builder + * Builder for star tree. Defines the algorithm to construct star-tree + * See {@link StarTreesBuilder} for information around the construction of star-trees based on star-tree fields + * * @opensearch.experimental */ -public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { +public abstract class BaseStarTreeBuilder implements StarTreeBuilder { private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); @@ -52,7 +53,7 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected final Set skipStarNodeCreationForDimensions; - protected final List metricAggregationDescriptors; + protected final List metricAggregatorInfos; protected final int numMetrics; protected final int numDimensions; protected int numStarTreeDocs; @@ -65,24 +66,21 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected DocIdSetIterator[] dimensionReaders; protected Map fieldProducerMap; - protected DocValuesConsumer docValuesConsumer; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; /** - * Constructor for base star-tree builder + * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * - * @param starTreeField holds the configuration for the star tree - * @param fieldProducerMap helps return the doc values iterator for each type based on field name - * @param docValuesConsumer to consume the new aggregated metrics during flush - * @param state stores the segment write state - * @param mapperService helps to find the original type of the field + * @param starTreeField holds the configuration for the star tree + * @param fieldProducerMap helps return the doc values iterator for each type based on field name + * @param state stores the segment write state + * @param mapperService helps to find the original type of the field */ protected BaseStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { @@ -91,7 +89,6 @@ protected BaseStarTreeBuilder( this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); - this.docValuesConsumer = docValuesConsumer; this.fieldProducerMap = fieldProducerMap; this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); @@ -109,7 +106,7 @@ protected BaseStarTreeBuilder( this.skipStarNodeCreationForDimensions.add(i); } FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); - DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); + DocValuesType dimensionDocValuesType = dimensionFieldInfos.getDocValuesType(); dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, @@ -117,50 +114,53 @@ protected BaseStarTreeBuilder( ); } - this.metricAggregationDescriptors = generateMetricStatFieldPairs(mapperService, state); - this.numMetrics = metricAggregationDescriptors.size(); + this.metricAggregatorInfos = generateMetricAggregatorInfos(mapperService, state); + this.numMetrics = metricAggregatorInfos.size(); this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); } /** - * Generates the MetricStatFieldPairs for all the metrics on a field + * Generates the configuration required to perform aggregation for all the metrics on a field * - * @return list of metric stat mapped with respective fields + * @return list of MetricAggregatorInfo */ - public List generateMetricStatFieldPairs(MapperService mapperService, SegmentWriteState state) + public List generateMetricAggregatorInfos(MapperService mapperService, SegmentWriteState state) throws IOException { - List metricAggregationDescriptors = new ArrayList<>(); - IndexNumericFieldData.NumericType numericType; - DocIdSetIterator metricStatReader = null; + List metricAggregatorInfos = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { + IndexNumericFieldData.NumericType numericType; + DocIdSetIterator metricStatReader = null; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); } else { - logger.error("metric mapper is not of type number field mapper"); - throw new IllegalStateException("metric mapper is not of type number field mapper"); + logger.error("unsupported mapper type"); + throw new IllegalStateException("unsupported mapper type"); } - // Ignore the column for COUNT aggregation function + + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); + DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); if (metricType != MetricStat.COUNT) { - FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); - DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); + // Need not initialize the metric reader for COUNT metric type metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( metricDocValuesType, metricFieldInfos, fieldProducerMap.get(metricFieldInfos.name) ); } - MetricAggregationDescriptor metricAggregationDescriptor = new MetricAggregationDescriptor( + + MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( metricType, metric.getField(), + starTreeField.getName(), numericType, metricStatReader ); - metricAggregationDescriptors.add(metricAggregationDescriptor); + metricAggregatorInfos.add(metricAggregatorInfo); } } - return metricAggregationDescriptors; + return metricAggregatorInfos; } /** @@ -185,7 +185,7 @@ public List generateMetricStatFieldPairs(MapperServ * * @return Star tree documents */ - public abstract List getStarTreeDocuments() throws IOException; + public abstract List getStarTreeDocuments(); /** * Returns the value of the dimension for the given dimension id and document in the star-tree. @@ -203,10 +203,10 @@ public List generateMetricStatFieldPairs(MapperServ * @param numDocs number of documents in the given segment * @return Iterator for the aggregated star-tree document */ - public abstract Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException; + public abstract Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException; /** - * Generates aggregated star-tree document for star-node. + * Generates aggregated star-tree documents for star-node. * * @param startDocId start document id (inclusive) in the star-tree * @param endDocId end document id (exclusive) in the star-tree @@ -218,6 +218,7 @@ public abstract Iterator generateStarTreeDocumentsForStarNode( /** * Returns the star-tree document from the segment + * * @throws IOException when we are unable to build a star tree document from the segment */ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { @@ -235,25 +236,18 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; for (int i = 0; i < numDimensions; i++) { - try { - dimensionReaders[i].nextDoc(); - } catch (IOException e) { - logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("dimension does not have an associated reader", e); - throw new IllegalStateException("dimension should have a valid associated reader", e); - } catch (Exception e) { - logger.error("unable to read the dimension values from the segment", e); - throw new IllegalStateException("unable to read the dimension values from the segment", e); - } + if (dimensionReaders[i] != null) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + throw e; + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); + } - if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { - dimensions[i] = handleDateDimension( - starTreeField.getDimensionsOrder().get(i).getField(), - starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]) - ); - } else { - dimensions[i] = starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); } } return dimensions; @@ -268,21 +262,18 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - // Ignore the column for COUNT aggregation function - DocIdSetIterator metricStatReader = metricAggregationDescriptors.get(i).getMetricStatReader(); + DocIdSetIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { metricStatReader.nextDoc(); } catch (IOException e) { logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("metric does not have an associated reader", e); - throw new IllegalStateException("metric should have a valid associated reader", e); + throw e; } catch (Exception e) { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextOrd(metricStatReader); + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricStatReader); } } return metrics; @@ -305,40 +296,62 @@ protected StarTreeDocument reduceSegmentStarTreeDocuments( Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - metrics[i] = metricValueAggregator.getInitialAggregatedValue((Long) segmentDocument.metrics[i], starTreeNumericType); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot parse initial segment doc value", e); - throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + metrics[i] = metricValueAggregator.getInitialAggregatedValueForSegmentDocValue( + getLong(segmentDocument.metrics[i]), + starTreeNumericType + ); } catch (Exception e) { logger.error("Cannot parse initial segment doc value", e); - throw new RuntimeException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - aggregatedSegmentDocument.metrics[i] = metricValueAggregator.applySegmentRawValue( + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + aggregatedSegmentDocument.metrics[i] = metricValueAggregator.mergeAggregatedValueAndSegmentValue( aggregatedSegmentDocument.metrics[i], - (Long) segmentDocument.metrics[i], + getLong(segmentDocument.metrics[i]), starTreeNumericType ); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot apply segment doc value for aggregation", e); - throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } catch (Exception e) { logger.error("Cannot apply segment doc value for aggregation", e); - throw new RuntimeException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } } return aggregatedSegmentDocument; } } + /** + * Safely converts the metric value of object type to long. + * + * @param metric value of the metric + * @return converted metric value to long + */ + private static long getLong(Object metric) { + + Long metricValue = null; + try { + if (metric instanceof Long) { + metricValue = (long) metric; + } else if (metric != null) { + metricValue = Long.valueOf(String.valueOf(metric)); + } + } catch (Exception e) { + throw new IllegalStateException("unable to cast segment metric", e); + } + + if (metricValue == null) { + throw new IllegalStateException("unable to cast segment metric"); + } + return metricValue; + } + /** * Merges a star-tree document into an aggregated star-tree document. * A new aggregated star-tree document is created if the aggregated document is null. @@ -354,30 +367,22 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = metricAggregationDescriptors.get(i).getValueAggregators().getAggregatedValue(starTreeDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot get aggregated value", e); - throw new IllegalArgumentException("Cannot get aggregated value[" + starTreeDocument.metrics[i] + "]"); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().getInitialAggregatedValue(starTreeDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot get value for aggregation", e); - throw new RuntimeException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedDocument.metrics[i] = metricAggregationDescriptors.get(i) + aggregatedDocument.metrics[i] = metricAggregatorInfos.get(i) .getValueAggregators() - .applyAggregatedValue(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new IllegalArgumentException( - "Cannot apply value to aggregated document for aggregation[" + starTreeDocument.metrics[i] + "]" - ); + .mergeAggregatedValues(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new RuntimeException( + throw new IllegalStateException( "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]" ); } @@ -393,9 +398,9 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum */ public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.debug("Star-tree build is a go with star tree field{}", starTreeField); + logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - Iterator starTreeDocumentIterator = sortMergeAndAggregateStarTreeDocument(totalSegmentDocs); + Iterator starTreeDocumentIterator = sortAndAggregateStarTreeDocument(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -465,9 +470,9 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { /** * Implements the algorithm to construct a star-tree * - * @param node star-tree node - * @param startDocId start document id - * @param endDocId end document id + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -498,8 +503,8 @@ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocI /** * Constructs non star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with non-star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -535,8 +540,8 @@ private Map constructNonStarNodes(int start /** * Constructs star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -576,7 +581,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (int i = node.startDocId; i < node.endDocId; i++) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } @@ -603,7 +607,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index 71eafc182c573..ae37cb3d29a68 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -208,6 +208,7 @@ public Void visit(Binding binding) { scopeInstancesInUse.put(scope, binding.getSource()); } } + return null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index b4837809b9706..193c66cfa033c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -17,7 +17,7 @@ * @opensearch.experimental */ public class CountValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -25,27 +25,27 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return 1.0; } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return value + 1; } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { return value + aggregatedValue; } @Override - public Double getAggregatedValue(Double value) { + public Double getInitialAggregatedValue(Double value) { return value; } @@ -58,8 +58,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -67,8 +67,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java similarity index 63% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index f5e1055b6ef24..03e3de0831ed6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -18,18 +18,11 @@ * Builds aggregation function and doc values field pair to support various aggregations * @opensearch.experimental */ -public class MetricAggregationDescriptor implements Comparable { - - public static final String DELIMITER = "__"; - public static final String STAR = "*"; - public static final MetricAggregationDescriptor COUNT_STAR = new MetricAggregationDescriptor( - MetricStat.COUNT, - STAR, - IndexNumericFieldData.NumericType.DOUBLE, - null - ); - - private final String metricStatName; +public class MetricAggregatorInfo implements Comparable { + + public static final String DELIMITER = "_"; + private final String metric; + private final String starFieldName; private final MetricStat metricStat; private final String field; private final ValueAggregator valueAggregators; @@ -37,11 +30,12 @@ public class MetricAggregationDescriptor implements Comparable o.field) - .thenComparing((MetricAggregationDescriptor o) -> o.metricStat) + public int compareTo(MetricAggregatorInfo other) { + return Comparator.comparing((MetricAggregatorInfo o) -> o.field) + .thenComparing((MetricAggregatorInfo o) -> o.metricStat) .compare(this, other); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index d9e55caae8784..5cbd233b2328e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -19,7 +19,8 @@ */ public class SumValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private CompensatedSum kahanSummation = new CompensatedSum(0, 0); @Override public MetricStat getAggregationType() { @@ -27,34 +28,36 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { - return starTreeNumericType.getDoubleValue(segmentDocValue); + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + kahanSummation.reset(0,0); + kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + return kahanSummation.value(); } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { - CompensatedSum kahanSummation = new CompensatedSum(0, 0); - kahanSummation.add(value); + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + assert kahanSummation.value() == value; kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); return kahanSummation.value(); } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { - CompensatedSum kahanSummation = new CompensatedSum(0, 0); + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { + assert kahanSummation.value() == aggregatedValue; kahanSummation.add(value); - kahanSummation.add(aggregatedValue); return kahanSummation.value(); } @Override - public Double getAggregatedValue(Double value) { - return value; + public Double getInitialAggregatedValue(Double value) { + kahanSummation.reset(0,0); + kahanSummation.add(value); + return kahanSummation.value(); } @Override @@ -66,8 +69,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -75,8 +78,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index 397f497d0105e..3dd1f85845c17 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -25,27 +25,27 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - StarTreeNumericType getStarTreeNumericType(); + StarTreeNumericType getAggregatedValueType(); /** * Returns the initial aggregated value. */ - A getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies a segment doc value to the current aggregated value. */ - A applySegmentRawValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A mergeAggregatedValueAndSegmentValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies an aggregated value to the current aggregated value. */ - A applyAggregatedValue(A value, A aggregatedValue); + A mergeAggregatedValues(A value, A aggregatedValue); /** * Clones an aggregated value. */ - A getAggregatedValue(A value); + A getInitialAggregatedValue(A value); /** * Returns the maximum size in bytes of the aggregated values seen so far. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 2bde2f16f91f8..4ee0b0b5b13f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -46,9 +46,9 @@ public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationT switch (aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: - return SumValueAggregator.STAR_TREE_NUMERIC_TYPE; + return SumValueAggregator.VALUE_AGGREGATOR_TYPE; case COUNT: - return CountValueAggregator.STAR_TREE_NUMERIC_TYPE; + return CountValueAggregator.VALUE_AGGREGATOR_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index 28bfd82c69fdf..f09fb1567cb26 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -20,7 +20,10 @@ public enum StarTreeNumericType { HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), - DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble); + DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), + INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), + SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), + UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); final IndexNumericFieldData.NumericType numericType; final Function converter; @@ -44,6 +47,12 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.LONG; case DOUBLE: return StarTreeNumericType.DOUBLE; + case INT: + return StarTreeNumericType.INT; + case SHORT: + return StarTreeNumericType.SHORT; + case UNSIGNED_LONG: + return StarTreeNumericType.UNSIGNED_LONG; default: throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index f3fdd0d1162c3..b840d9436a26a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -10,6 +10,7 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.annotation.ExperimentalApi; /** @@ -31,7 +32,20 @@ public static double longToDouble(Long value) { return (double) value; } + public static double intToDouble(Long value) { + return (double) value; + } + + public static double shortToDouble(Long value) { + return (double) value; + } + public static Double sortableLongtoDouble(Long value) { return NumericUtils.sortableLongToDouble(value); } + + public static double unsignedlongToDouble(Long value) { + return Numbers.unsignedLongToDouble(value); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java similarity index 89% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 0289a56ce5b5f..86ec912c87668 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; @@ -28,28 +27,26 @@ * @opensearch.experimental */ @ExperimentalApi -public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { +public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); /** - * Constructor for OnHeapSingleTreeBuilder + * Constructor for OnHeapStarTreeBuilder * * @param starTreeField star-tree field * @param fieldProducerMap helps with document values producer for a particular field - * @param docValuesConsumer document values consumer * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field * @throws IOException throws an exception we are unable to construct an onheap star-tree */ - public OnHeapSingleTreeBuilder( + public OnHeapStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - super(starTreeField, fieldProducerMap, docValuesConsumer, segmentWriteState, mapperService); + super(starTreeField, fieldProducerMap, segmentWriteState, mapperService); } @Override @@ -63,7 +60,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return starTreeDocuments; } @@ -74,12 +71,12 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { starTreeDocuments[i] = getSegmentStarTreeDocument(); } - return sortMergeAndAggregateStarTreeDocument(starTreeDocuments); + return sortAndAggregateStarTreeDocument(starTreeDocuments); } /** @@ -88,7 +85,7 @@ public Iterator sortMergeAndAggregateStarTreeDocument(int numD * @return iterator for star-tree documents * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - public Iterator sortMergeAndAggregateStarTreeDocument(StarTreeDocument[] starTreeDocuments) throws IOException { + public Iterator sortAndAggregateStarTreeDocument(StarTreeDocument[] starTreeDocuments) throws IOException { // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 44866caba938b..ef542a1c848f2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -18,11 +18,11 @@ * @opensearch.experimental */ @ExperimentalApi -public interface SingleTreeBuilder extends Closeable { +public interface StarTreeBuilder extends Closeable { /** * Builds the star tree based on star-tree field * @throws IOException when we are unable to build star-tree */ - void build() throws Exception; + void build() throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index e4cf2a45e131b..7c035ed6f21be 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; @@ -31,8 +30,6 @@ public class StarTreeDocValuesIteratorAdapter { */ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { - case SORTED_SET: - return producer.getSortedSet(field); case SORTED_NUMERIC: return producer.getSortedNumeric(field); default: @@ -41,12 +38,10 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field } /** - * Returns the next ordinal for the given iterator + * Returns the next value for the given iterator */ - public long getNextOrd(DocIdSetIterator iterator) throws IOException { - if (iterator instanceof SortedSetDocValues) { - return ((SortedSetDocValues) iterator).nextOrd(); - } else if (iterator instanceof SortedNumericDocValues) { + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedNumericDocValues) { return ((SortedNumericDocValues) iterator).nextValue(); } else { throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); @@ -54,7 +49,8 @@ public long getNextOrd(DocIdSetIterator iterator) throws IOException { } /** - * Returns the doc id for the next document from the given iterator + * Moves to the next doc in the iterator + * Returns the doc id for the next document from the given iterator */ public int nextDoc(DocIdSetIterator iterator) throws IOException { return iterator.nextDoc(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 4d110609462a5..65f6551cd2523 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -10,22 +10,23 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.StarTreeMapper; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; /** - * Builder to construct star tree based on multiple star tree configs + * Builder to construct star-trees based on multiple star-tree fields. * * @opensearch.experimental */ @@ -35,61 +36,52 @@ public class StarTreesBuilder implements Closeable { private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class); private final List starTreeFields; - private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; - private final DocValuesConsumer docValuesConsumer; private final SegmentWriteState state; private final Map fieldProducerMap; private final MapperService mapperService; public StarTreesBuilder( - List starTreeFields, - StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) { + List starTreeFields = new ArrayList<>(); + for (CompositeMappedFieldType compositeMappedFieldType : mapperService.getCompositeFieldTypes()) { + if (compositeMappedFieldType instanceof StarTreeMapper.StarTreeFieldType) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) compositeMappedFieldType; + starTreeFields.add(new StarTreeField(starTreeFieldType.name(), starTreeFieldType.getDimensions(), starTreeFieldType.getMetrics(), starTreeFieldType.getStarTreeConfig())); + } + } + this.starTreeFields = starTreeFields; - if (starTreeFields == null || starTreeFields.isEmpty()) { - throw new IllegalArgumentException("Must provide star-tree builder configs"); + if (starTreeFields.isEmpty()) { + throw new IllegalArgumentException("Must provide star-tree field to build star trees"); } - this.buildMode = buildMode; this.fieldProducerMap = fieldProducerMap; - this.docValuesConsumer = docValuesConsumer; this.state = segmentWriteState; this.mapperService = mapperService; + } /** * Builds the star-trees. */ - public void build() throws Exception { + public void build() throws IOException { long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); - logger.debug("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + logger.debug("Starting building {} star-trees with star-tree fields", numStarTrees); // Build all star-trees for (int i = 0; i < numStarTrees; i++) { StarTreeField starTreeField = starTreeFields.get(i); - try ( - SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( - starTreeField, - buildMode, - fieldProducerMap, - docValuesConsumer, - state, - mapperService - ) - ) { - singleTreeBuilder.build(); + try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(starTreeField, fieldProducerMap, state, mapperService)) { + starTreeBuilder.build(); } } logger.debug( - "Took {} ms to building {} star-trees with configs: {} using {} builder", + "Took {} ms to building {} star-trees with star-tree fields", System.currentTimeMillis() - startTime, - numStarTrees, - starTreeFields, - buildMode + numStarTrees ); } @@ -98,20 +90,18 @@ public void close() throws IOException { } - private static SingleTreeBuilder getSingleTreeBuilder( + private static StarTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, - StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { - switch (buildMode) { + switch (starTreeField.getStarTreeConfig().getBuildMode()) { case ON_HEAP: - return new OnHeapSingleTreeBuilder(starTreeField, fieldProducerMap, docValuesConsumer, state, mapperService); + return new OnHeapStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService); default: throw new IllegalArgumentException( - String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) + String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", starTreeField.getStarTreeConfig().getBuildMode()) ); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 656196c7ffe0f..96e6681d40a76 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -22,15 +22,50 @@ private StarTreeBuilderUtils() {} public static final int ALL = -1; - /** Tree node representation */ + /** + * Represents a node in a tree data structure, specifically designed for a star-tree implementation. + * A star-tree node will represent both star and non-star nodes. + */ public static class TreeNode { + + /** + * The dimension id for the dimension (field) associated with this star-tree node. + */ public int dimensionId = ALL; + + /** + * The starting document id (inclusive) associated with this star-tree node. + */ public int startDocId = ALL; + + /** + * The ending document id (exclusive) associated with this star-tree node. + */ public int endDocId = ALL; + + /** + * The aggregated document id associated with this star-tree node. + */ public int aggregatedDocId = ALL; + + /** + * The child dimension identifier associated with this star-tree node. + */ public int childDimensionId = ALL; + + /** + * The value of the dimension associated with this star-tree node. + */ public long dimensionValue = ALL; + + /** + * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions). + */ public boolean isStarNode = false; + + /** + * A map containing the child nodes of this star-tree node, keyed by their dimension id. + */ public Map children; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java new file mode 100644 index 0000000000000..e7d659ee177f3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.test.OpenSearchTestCase; + +public class CountValueAggregatorTests extends OpenSearchTestCase { + private final CountValueAggregator aggregator = new CountValueAggregator(); + + public void testGetAggregationType() { + assertEquals(MetricStat.COUNT.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + assertEquals(CountValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValueAndSegmentValue() { + assertEquals(3.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValues() { + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); + } + + public void testGetInitialAggregatedValue() { + assertEquals(3.0, aggregator.getInitialAggregatedValue(3.0), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testToLongValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.doubleToSortableLong(3.0), aggregator.toLongValue(3.0), 0.0); + } + + public void testToStarTreeNumericTypeValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java similarity index 63% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 98b79141df9eb..d08f637a3f0a9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -12,12 +12,13 @@ import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.test.OpenSearchTestCase; -public class MetricAggregationDescriptorTests extends OpenSearchTestCase { +public class MetricAggregatorInfoTests extends OpenSearchTestCase { public void testConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -26,57 +27,66 @@ public void testConstructor() { } public void testCountStarConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.COUNT, "anything", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); + assertEquals("anything", pair.getField()); } public void testToFieldName() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - assertEquals("sum__column2", pair.toFieldName()); + assertEquals("star_tree_field_column2_sum", pair.toFieldName()); } public void testEquals() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregationDescriptor(MetricStat.COUNT, "column1", IndexNumericFieldData.NumericType.DOUBLE, null) + new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) + ); + assertNotEquals( + pair1, + new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertNotEquals(pair1, new MetricAggregationDescriptor(MetricStat.SUM, "column2", IndexNumericFieldData.NumericType.DOUBLE, null)); } public void testHashCode() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -84,21 +94,24 @@ public void testHashCode() { } public void testCompareTo() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair3 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair3 = new MetricAggregatorInfo( MetricStat.COUNT, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 287deeead9d9d..dc55f85aadd4d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -9,38 +9,56 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; +import org.junit.Before; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; public class SumValueAggregatorTests extends OpenSearchTestCase { - private final SumValueAggregator aggregator = new SumValueAggregator(); + private SumValueAggregator aggregator; + + @Before + public void setup(){ + aggregator = new SumValueAggregator(); + } public void testGetAggregationType() { assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); } - public void testGetStarTreeNumericType() { - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, aggregator.getStarTreeNumericType()); + public void testGetAggregatedValueType() { + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); } - public void testGetInitialAggregatedValue() { - assertEquals(1.0, aggregator.getInitialAggregatedValue(1L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.getInitialAggregatedValue(null, StarTreeNumericType.DOUBLE)); + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(1L, StarTreeNumericType.LONG), 0.0); + assertThrows( + NullPointerException.class, + () -> aggregator.getInitialAggregatedValueForSegmentDocValue(null, StarTreeNumericType.DOUBLE) + ); } - public void testApplySegmentRawValue() { - assertEquals(5.0, aggregator.applySegmentRawValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.applySegmentRawValue(3.14, null, StarTreeNumericType.DOUBLE)); + public void testMergeAggregatedValueAndSegmentValue() { + aggregator.getInitialAggregatedValue(2.0); + assertEquals(5.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); } - public void testApplyAggregatedValue() { - assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { + aggregator.getInitialAggregatedValue(2.0); + assertThrows( + NullPointerException.class, + () -> aggregator.mergeAggregatedValueAndSegmentValue(2.0, null, StarTreeNumericType.LONG) + ); } - public void testGetAggregatedValue() { - assertEquals(3.14, aggregator.getAggregatedValue(3.14), 0.0); + public void testMergeAggregatedValues() { + aggregator.getInitialAggregatedValue(3.0); + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); + } + + public void testGetInitialAggregatedValue() { + assertEquals(3.14, aggregator.getInitialAggregatedValue(3.14), 0.0); } public void testGetMaxAggregatedValueByteSize() { @@ -48,12 +66,10 @@ public void testGetMaxAggregatedValueByteSize() { } public void testToLongValue() { - SumValueAggregator aggregator = new SumValueAggregator(); assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.toLongValue(3.14), 0.0); } public void testToStarTreeNumericTypeValue() { - SumValueAggregator aggregator = new SumValueAggregator(); assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index 21e2d1304f085..ce61ab839cc61 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -20,8 +20,8 @@ public void testGetValueAggregatorForSumType() { assertEquals(SumValueAggregator.class, aggregator.getClass()); } - public void testGetStarTreeNumericTypeForSumType() { + public void testGetAggregatedValueTypeForSumType() { StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, starTreeNumericType); + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, starTreeNumericType); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 7a13ab7701912..e96295d5b619d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -30,7 +30,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; @@ -75,6 +75,7 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static Directory directory; private static FieldInfo[] fieldsInfo; private static SegmentWriteState state; + private static StarTreeField starTreeField; @BeforeClass public static void setup() throws IOException { @@ -82,7 +83,7 @@ public static void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); - StarTreeField compositeField = new StarTreeField( + starTreeField = new StarTreeField( "test", dimensionsOrder, metrics, @@ -150,7 +151,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -160,7 +161,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return List.of(); } @@ -170,7 +171,7 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException { return null; } @@ -182,13 +183,13 @@ public Iterator generateStarTreeDocumentsForStarNode(int start }; } - public void test_generateMetricStatFieldPairs() throws IOException { - List metricAggregationDescriptors = builder.generateMetricStatFieldPairs(mapperService, state); - List expectedMetricAggregationDescriptors = List.of( - new MetricAggregationDescriptor(MetricStat.SUM, "field2", IndexNumericFieldData.NumericType.DOUBLE, null), - new MetricAggregationDescriptor(MetricStat.SUM, "field4", IndexNumericFieldData.NumericType.DOUBLE, null) + public void test_generateMetricAggregatorInfos() throws IOException { + List metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService, state); + List expectedMetricAggregatorInfos = List.of( + new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null), + new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertEquals(metricAggregationDescriptors, expectedMetricAggregationDescriptors); + assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos); } public void test_reduceStarTreeDocuments() { @@ -196,10 +197,11 @@ public void test_reduceStarTreeDocuments() { StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); - StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(starTreeDocument1, starTreeDocument2); + StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(null, starTreeDocument1); + StarTreeDocument resultStarTreeDocument = builder.reduceStarTreeDocuments(mergedStarTreeDocument, starTreeDocument2); - assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); - assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); + assertEquals(resultStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); + assertEquals(resultStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); } @Override diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java similarity index 77% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 73955080c36ae..e3e663f99994a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValuesType; @@ -53,9 +52,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { +public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { - private OnHeapSingleTreeBuilder builder; + private OnHeapStarTreeBuilder builder; private MapperService mapperService; private List dimensionsOrder; private List fields = List.of( @@ -75,15 +74,17 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { private FieldInfo[] fieldsInfo; private StarTreeField compositeField; private Map fieldProducerMap; - private DocValuesConsumer docValuesConsumer; private SegmentWriteState writeState; @Before public void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); - metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.COUNT)) + ); - docValuesConsumer = mock(DocValuesConsumer.class); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); compositeField = new StarTreeField( @@ -143,31 +144,33 @@ public void setup() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); } - public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { + public void test_sortAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -175,10 +178,11 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); int numOfAggregatedDocuments = 0; @@ -192,6 +196,7 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -200,17 +205,17 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { } - public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOException { + public void test_sortAndAggregateStarTreeDocument_nullMetric() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); - StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null, randomDouble() }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -218,10 +223,11 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc Long metric2 = starTreeDocuments[i].metrics[1] != null ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) : null; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2 }); + Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); @@ -241,20 +247,20 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc } - public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { + public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -262,10 +268,11 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); int numOfAggregatedDocuments = 0; @@ -279,6 +286,7 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -292,15 +300,15 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -308,10 +316,11 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); int numOfAggregatedDocuments = 0; @@ -325,6 +334,7 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -343,38 +353,40 @@ public void test_build_halfFloatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; starTreeDocuments[0] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10) } + new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[1] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6) } + new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[2] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12) } + new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[3] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4) } + new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[4] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16) } + new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16), new HalfFloatPoint("field6", 10) } ); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -385,10 +397,13 @@ public void test_build_halfFloatMetrics() throws IOException { long metric2 = HalfFloatPoint.halfFloatToSortableShort( ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() ); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); builder.build(segmentStarTreeDocumentIterator); @@ -410,33 +425,36 @@ public void test_build_floatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F, randomFloat() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F, randomFloat() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F, randomFloat() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F, randomFloat() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F, randomFloat() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); builder.build(segmentStarTreeDocumentIterator); @@ -458,33 +476,36 @@ public void test_build_longMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L, randomLong() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L, randomLong() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L, randomLong() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L, randomLong() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L, randomLong() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = (Long) starTreeDocuments[i].metrics[0]; long metric2 = (Long) starTreeDocuments[i].metrics[1]; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); builder.build(segmentStarTreeDocumentIterator); @@ -498,14 +519,14 @@ public void test_build_longMetrics() throws IOException { private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), - new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }), + new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }) ); return expectedStarTreeDocuments.iterator(); } @@ -515,20 +536,21 @@ public void test_build() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( segmentStarTreeDocuments ); builder.build(segmentStarTreeDocumentIterator); @@ -555,6 +577,7 @@ private static void assertStarTreeDocuments( assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index f5d53afb8f774..59cadfa737786 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; @@ -56,14 +55,6 @@ public static void setup() { ); } - public void testCreateIterator_SortedSet() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); - assertEquals(iterator.getClass(), result.getClass()); - } - public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); @@ -80,26 +71,18 @@ public void testCreateIterator_UnsupportedType() { assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } - public void testGetNextValue_SortedSet() throws IOException { - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(iterator.nextOrd()).thenReturn(42L); - - long result = factory.getNextOrd(iterator); - assertEquals(42L, result); - } - public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextValue()).thenReturn(123L); - long result = factory.getNextOrd(iterator); + long result = factory.getNextValue(iterator); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextOrd(iterator); }); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); } From af31c4b5d5f13ddb3d013a29b6ae5750091b54da Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 28 Jun 2024 14:35:21 +0530 Subject: [PATCH 05/16] handling for missing doc values Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 49 +++--- .../datacube/startree/StarTreeDocument.java | 4 +- .../aggregators/MetricAggregatorInfo.java | 11 +- .../aggregators/SumValueAggregator.java | 4 +- .../builder/OnHeapStarTreeBuilder.java | 6 +- .../StarTreeDocValuesIteratorAdapter.java | 39 ++++- .../startree/builder/StarTreesBuilder.java | 21 ++- .../utils/CoordinatedDocumentReader.java | 90 +++++++++++ .../aggregators/SumValueAggregatorTests.java | 9 +- .../builder/BaseStarTreeBuilderTests.java | 6 +- .../builder/OnHeapStarTreeBuilderTests.java | 142 ++++++++---------- .../StarTreeValuesIteratorFactoryTests.java | 55 +++++-- 12 files changed, 288 insertions(+), 148 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index af43dab7c498b..728b630da3efd 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -23,6 +22,7 @@ import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; +import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -49,6 +49,9 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); + /** + * Default value for star node + */ public static final int STAR_IN_DOC_VALUES_INDEX = -1; protected final Set skipStarNodeCreationForDimensions; @@ -63,11 +66,11 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); - protected DocIdSetIterator[] dimensionReaders; + protected CoordinatedDocumentReader[] dimensionReaders; protected Map fieldProducerMap; - private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; + private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorAdapter; private final StarTreeField starTreeField; /** @@ -90,14 +93,14 @@ protected BaseStarTreeBuilder( this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); this.fieldProducerMap = fieldProducerMap; - this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); + this.starTreeDocValuesIteratorAdapter = new StarTreeDocValuesIteratorAdapter(); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); this.numDimensions = dimensionsSplitOrder.size(); this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = state.segmentInfo.maxDoc(); - this.dimensionReaders = new DocIdSetIterator[numDimensions]; + this.dimensionReaders = new CoordinatedDocumentReader[numDimensions]; Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); for (int i = 0; i < numDimensions; i++) { @@ -107,7 +110,7 @@ protected BaseStarTreeBuilder( } FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); DocValuesType dimensionDocValuesType = dimensionFieldInfos.getDocValuesType(); - dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( + dimensionReaders[i] = starTreeDocValuesIteratorAdapter.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, fieldProducerMap.get(dimensionFieldInfos.name) @@ -130,7 +133,7 @@ public List generateMetricAggregatorInfos(MapperService ma for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { IndexNumericFieldData.NumericType numericType; - DocIdSetIterator metricStatReader = null; + CoordinatedDocumentReader metricStatReader = null; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); @@ -143,7 +146,7 @@ public List generateMetricAggregatorInfos(MapperService ma DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); if (metricType != MetricStat.COUNT) { // Need not initialize the metric reader for COUNT metric type - metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( + metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( metricDocValuesType, metricFieldInfos, fieldProducerMap.get(metricFieldInfos.name) @@ -221,9 +224,9 @@ public abstract Iterator generateStarTreeDocumentsForStarNode( * * @throws IOException when we are unable to build a star tree document from the segment */ - protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { - long[] dimensions = getStarTreeDimensionsFromSegment(); - Object[] metrics = getStarTreeMetricsFromSegment(); + protected StarTreeDocument getSegmentStarTreeDocument(int currentDocId) throws IOException { + Long[] dimensions = getStarTreeDimensionsFromSegment(currentDocId); + Object[] metrics = getStarTreeMetricsFromSegment(currentDocId); return new StarTreeDocument(dimensions, metrics); } @@ -233,12 +236,12 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { * @return dimension values for each of the star-tree dimension * @throws IOException when we are unable to iterate to the next doc for the given dimension readers */ - long[] getStarTreeDimensionsFromSegment() throws IOException { - long[] dimensions = new long[numDimensions]; + Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { + Long[] dimensions = new Long[numDimensions]; for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { try { - dimensionReaders[i].nextDoc(); + starTreeDocValuesIteratorAdapter.nextDoc(dimensionReaders[i], currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw e; @@ -247,7 +250,7 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { throw new IllegalStateException("unable to read the dimension values from the segment", e); } - dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorAdapter.getNextValue(dimensionReaders[i], currentDocId); } } return dimensions; @@ -259,13 +262,13 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { * @return metric values for each of the star-tree metric * @throws IOException when we are unable to iterate to the next doc for the given metric readers */ - private Object[] getStarTreeMetricsFromSegment() throws IOException { + private Object[] getStarTreeMetricsFromSegment(int currentDocId) throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - DocIdSetIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); + CoordinatedDocumentReader metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { - metricStatReader.nextDoc(); + starTreeDocValuesIteratorAdapter.nextDoc(metricStatReader, currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw e; @@ -273,7 +276,7 @@ private Object[] getStarTreeMetricsFromSegment() throws IOException { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricStatReader); + metrics[i] = starTreeDocValuesIteratorAdapter.getNextValue(metricStatReader, currentDocId); } } return metrics; @@ -292,7 +295,7 @@ protected StarTreeDocument reduceSegmentStarTreeDocuments( StarTreeDocument segmentDocument ) { if (aggregatedSegmentDocument == null) { - long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); + Long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { @@ -363,7 +366,7 @@ private static long getLong(Object metric) { public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocument, StarTreeDocument starTreeDocument) { // aggregate the documents if (aggregatedDocument == null) { - long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); + Long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { @@ -585,7 +588,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } for (int i = node.dimensionId + 1; i < numDimensions; i++) { - aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); } node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); @@ -611,7 +614,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } for (int i = node.dimensionId + 1; i < numDimensions; i++) { - aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); } node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java index 3d537be1c3227..0fc66b27e65a9 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java @@ -18,10 +18,10 @@ */ @ExperimentalApi public class StarTreeDocument { - public final long[] dimensions; + public final Long[] dimensions; public final Object[] metrics; - public StarTreeDocument(long[] dimensions, Object[] metrics) { + public StarTreeDocument(Long[] dimensions, Object[] metrics) { this.dimensions = dimensions; this.metrics = metrics; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index 03e3de0831ed6..d2ae3e11c9d7b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -7,12 +7,13 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; import org.opensearch.index.fielddata.IndexNumericFieldData; import java.util.Comparator; +import java.util.Objects; /** * Builds aggregation function and doc values field pair to support various aggregations @@ -27,7 +28,7 @@ public class MetricAggregatorInfo implements Comparable { private final String field; private final ValueAggregator valueAggregators; private final StarTreeNumericType starTreeNumericType; - private final DocIdSetIterator metricStatReader; + private final CoordinatedDocumentReader metricStatReader; /** * Constructor for MetricAggregatorInfo @@ -37,7 +38,7 @@ public MetricAggregatorInfo( String field, String starFieldName, IndexNumericFieldData.NumericType numericType, - DocIdSetIterator metricStatReader + CoordinatedDocumentReader metricStatReader ) { this.metricStat = metricStat; this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat); @@ -86,7 +87,7 @@ public StarTreeNumericType getAggregatedValueType() { /** * @return metric value reader iterator */ - public DocIdSetIterator getMetricStatReader() { + public CoordinatedDocumentReader getMetricStatReader() { return metricStatReader; } @@ -99,7 +100,7 @@ public String toFieldName() { @Override public int hashCode() { - return 31 * metricStat.hashCode() + field.hashCode(); + return Objects.hashCode(toFieldName()); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index 5cbd233b2328e..663b404e380db 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -34,7 +34,7 @@ public StarTreeNumericType getAggregatedValueType() { @Override public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { - kahanSummation.reset(0,0); + kahanSummation.reset(0, 0); kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); return kahanSummation.value(); } @@ -55,7 +55,7 @@ public Double mergeAggregatedValues(Double value, Double aggregatedValue) { @Override public Double getInitialAggregatedValue(Double value) { - kahanSummation.reset(0,0); + kahanSummation.reset(0, 0); kahanSummation.add(value); return kahanSummation.value(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 86ec912c87668..be9304db92173 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -73,8 +73,8 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { @Override public Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; - for (int i = 0; i < numDocs; i++) { - starTreeDocuments[i] = getSegmentStarTreeDocument(); + for (int currentDocId = 0; currentDocId < numDocs; currentDocId++) { + starTreeDocuments[currentDocId] = getSegmentStarTreeDocument(currentDocId); } return sortAndAggregateStarTreeDocument(starTreeDocuments); } @@ -182,7 +182,7 @@ public boolean hasNext() { @Override public StarTreeDocument next() { StarTreeDocument next = reduceStarTreeDocuments(null, currentStarTreeDocument); - next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; + next.dimensions[dimensionId] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); while (docId < numDocs) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; if (!hasSameDimensions(starTreeDocument, currentStarTreeDocument)) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index 7c035ed6f21be..8a4bc0f7f0d59 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; import java.io.IOException; @@ -28,10 +29,11 @@ public class StarTreeDocValuesIteratorAdapter { /** * Creates an iterator for the given doc values type and field using the doc values producer */ - public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + public CoordinatedDocumentReader getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) + throws IOException { switch (type) { case SORTED_NUMERIC: - return producer.getSortedNumeric(field); + return new CoordinatedDocumentReader(producer.getSortedNumeric(field)); default: throw new IllegalArgumentException("Unsupported DocValuesType: " + type); } @@ -40,11 +42,27 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field /** * Returns the next value for the given iterator */ - public long getNextValue(DocIdSetIterator iterator) throws IOException { - if (iterator instanceof SortedNumericDocValues) { - return ((SortedNumericDocValues) iterator).nextValue(); + public Long getNextValue(CoordinatedDocumentReader coordinatedDocumentReader, int currentDocId) throws IOException { + if (coordinatedDocumentReader.getDocIdSetIterator() instanceof SortedNumericDocValues) { + SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) coordinatedDocumentReader.getDocIdSetIterator(); + if (coordinatedDocumentReader.getDocId() < 0 || coordinatedDocumentReader.getDocId() == DocIdSetIterator.NO_MORE_DOCS) { + throw new IllegalStateException("invalid doc id to fetch the next value"); + } + + if (coordinatedDocumentReader.getDocValue() == null) { + coordinatedDocumentReader.setDocValue(sortedNumericDocValues.nextValue()); + return coordinatedDocumentReader.getDocValue(); + } + + if (coordinatedDocumentReader.getDocId() == currentDocId) { + Long nextValue = coordinatedDocumentReader.getDocValue(); + coordinatedDocumentReader.setDocValue(null); + return nextValue; + } else { + return null; + } } else { - throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); + throw new IllegalStateException("Unsupported Iterator: " + coordinatedDocumentReader.getDocIdSetIterator().toString()); } } @@ -52,8 +70,13 @@ public long getNextValue(DocIdSetIterator iterator) throws IOException { * Moves to the next doc in the iterator * Returns the doc id for the next document from the given iterator */ - public int nextDoc(DocIdSetIterator iterator) throws IOException { - return iterator.nextDoc(); + public int nextDoc(CoordinatedDocumentReader iterator, int currentDocId) throws IOException { + if (iterator.getDocValue() != null) { + return iterator.getDocId(); + } + iterator.setDocId(iterator.getDocIdSetIterator().nextDoc()); + iterator.setDocValue(this.getNextValue(iterator, currentDocId)); + return iterator.getDocId(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 65f6551cd2523..6665ea59c4060 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -49,7 +49,14 @@ public StarTreesBuilder( for (CompositeMappedFieldType compositeMappedFieldType : mapperService.getCompositeFieldTypes()) { if (compositeMappedFieldType instanceof StarTreeMapper.StarTreeFieldType) { StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) compositeMappedFieldType; - starTreeFields.add(new StarTreeField(starTreeFieldType.name(), starTreeFieldType.getDimensions(), starTreeFieldType.getMetrics(), starTreeFieldType.getStarTreeConfig())); + starTreeFields.add( + new StarTreeField( + starTreeFieldType.name(), + starTreeFieldType.getDimensions(), + starTreeFieldType.getMetrics(), + starTreeFieldType.getStarTreeConfig() + ) + ); } } @@ -78,11 +85,7 @@ public void build() throws IOException { starTreeBuilder.build(); } } - logger.debug( - "Took {} ms to building {} star-trees with star-tree fields", - System.currentTimeMillis() - startTime, - numStarTrees - ); + logger.debug("Took {} ms to building {} star-trees with star-tree fields", System.currentTimeMillis() - startTime, numStarTrees); } @Override @@ -101,7 +104,11 @@ private static StarTreeBuilder getSingleTreeBuilder( return new OnHeapStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService); default: throw new IllegalArgumentException( - String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", starTreeField.getStarTreeConfig().getBuildMode()) + String.format( + Locale.ROOT, + "No star tree implementation is available for [%s] build mode", + starTreeField.getStarTreeConfig().getBuildMode() + ) ); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java new file mode 100644 index 0000000000000..0026cb7163235 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Coordinates the reading of documents across multiple DocIdSetIterators. + * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. + * @opensearch.experimental + */ +@ExperimentalApi +public class CoordinatedDocumentReader { + + /** + * The doc id set iterator associated for each field. + */ + private final DocIdSetIterator docIdSetIterator; + + /** + * The value associated with the latest document. + */ + private Long docValue; + + /** + * The id of the latest document. + */ + private int docId; + + /** + * Constructs a new CoordinatedDocumentReader instance with the given DocIdSetIterator. + * + * @param docIdSetIterator the DocIdSetIterator to be associated with this instance + */ + public CoordinatedDocumentReader(DocIdSetIterator docIdSetIterator) { + this.docIdSetIterator = docIdSetIterator; + } + + /** + * Returns the value associated with the latest document. + * + * @return the value associated with the latest document + */ + public Long getDocValue() { + return docValue; + } + + /** + * Sets the value associated with the latest document. + * + * @param docValue the value to be associated with the latest document + */ + public void setDocValue(Long docValue) { + this.docValue = docValue; + } + + /** + * Returns the id of the latest document. + * + * @return the id of the latest document + */ + public int getDocId() { + return docId; + } + + /** + * Sets the id of the latest document. + * + * @param docId the ID of the latest document + */ + public void setDocId(int docId) { + this.docId = docId; + } + + /** + * Returns the DocIdSetIterator associated with this instance. + * + * @return the DocIdSetIterator associated with this instance + */ + public DocIdSetIterator getDocIdSetIterator() { + return docIdSetIterator; + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index dc55f85aadd4d..3fb627e7cd434 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -9,17 +9,17 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.apache.lucene.util.NumericUtils; -import org.junit.Before; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; public class SumValueAggregatorTests extends OpenSearchTestCase { private SumValueAggregator aggregator; @Before - public void setup(){ + public void setup() { aggregator = new SumValueAggregator(); } @@ -46,10 +46,7 @@ public void testMergeAggregatedValueAndSegmentValue() { public void testMergeAggregatedValueAndSegmentValue_nullSegmentDocValue() { aggregator.getInitialAggregatedValue(2.0); - assertThrows( - NullPointerException.class, - () -> aggregator.mergeAggregatedValueAndSegmentValue(2.0, null, StarTreeNumericType.LONG) - ); + assertThrows(NullPointerException.class, () -> aggregator.mergeAggregatedValueAndSegmentValue(2.0, null, StarTreeNumericType.LONG)); } public void testMergeAggregatedValues() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index e96295d5b619d..9bbcd1546f3cd 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -193,10 +193,10 @@ public void test_generateMetricAggregatorInfos() throws IOException { } public void test_reduceStarTreeDocuments() { - StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); - StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + StarTreeDocument starTreeDocument1 = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 4.0, 8.0 }); + StarTreeDocument starTreeDocument2 = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 10.0, 6.0 }); - StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); + StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new Long[] { 1L, 3L, 5L, 8L }, new Double[] { 14.0, 14.0 }); StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(null, starTreeDocument1); StarTreeDocument resultStarTreeDocument = builder.reduceStarTreeDocuments(mergedStarTreeDocument, starTreeDocument2); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index e3e663f99994a..a13ad545f65af 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -162,15 +162,15 @@ public void test_sortAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -182,9 +182,7 @@ public void test_sortAndAggregateStarTreeDocument() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -210,12 +208,12 @@ public void test_sortAndAggregateStarTreeDocument_nullMetric() throws IOExceptio int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null, randomDouble() }); - StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, null, randomDouble() }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -227,9 +225,7 @@ public void test_sortAndAggregateStarTreeDocument_nullMetric() throws IOExceptio segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); @@ -252,15 +248,15 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) + new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -272,9 +268,7 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -300,15 +294,15 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -320,9 +314,7 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -369,23 +361,23 @@ public void test_build_halfFloatMetrics() throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; starTreeDocuments[0] = new StarTreeDocument( - new long[] { 2, 4, 3, 4 }, + new Long[] { 2L, 4L, 3L, 4L }, new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[1] = new StarTreeDocument( - new long[] { 3, 4, 2, 1 }, + new Long[] { 3L, 4L, 2L, 1L }, new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[2] = new StarTreeDocument( - new long[] { 3, 4, 2, 1 }, + new Long[] { 3L, 4L, 2L, 1L }, new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[3] = new StarTreeDocument( - new long[] { 2, 4, 3, 4 }, + new Long[] { 2L, 4L, 3L, 4L }, new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[4] = new StarTreeDocument( - new long[] { 3, 4, 2, 1 }, + new Long[] { 3L, 4L, 2L, 1L }, new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16), new HalfFloatPoint("field6", 10) } ); @@ -403,9 +395,7 @@ public void test_build_halfFloatMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -440,11 +430,11 @@ public void test_build_floatMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F, randomFloat() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F, randomFloat() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F, randomFloat() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F, randomFloat() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F, randomFloat() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Float[] { 12.0F, 10.0F, randomFloat() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 10.0F, 6.0F, randomFloat() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 14.0F, 12.0F, randomFloat() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Float[] { 9.0F, 4.0F, randomFloat() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Float[] { 11.0F, 16.0F, randomFloat() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -454,9 +444,7 @@ public void test_build_floatMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -491,11 +479,11 @@ public void test_build_longMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L, randomLong() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L, randomLong() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L, randomLong() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L, randomLong() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L, randomLong() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 12L, 10L, randomLong() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 10L, 6L, randomLong() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 14L, 12L, randomLong() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Long[] { 9L, 4L, randomLong() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Long[] { 11L, 16L, randomLong() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -505,9 +493,7 @@ public void test_build_longMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -519,14 +505,14 @@ public void test_build_longMetrics() throws IOException { private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }), - new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new Long[] { -1L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Double[] { 56.0, 48.0, 5.0 }), + new StarTreeDocument(new Long[] { -1L, -1L, -1L, -1L }, new Double[] { 56.0, 48.0, 5.0 }) ); return expectedStarTreeDocuments.iterator(); } @@ -536,11 +522,11 @@ public void test_build() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -550,9 +536,7 @@ public void test_build() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument( - segmentStarTreeDocuments - ); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index 59cadfa737786..4e865c4580fc1 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -28,12 +29,12 @@ public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase { - private static StarTreeDocValuesIteratorAdapter factory; + private static StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorAdapter; private static FieldInfo mockFieldInfo; @BeforeClass public static void setup() { - factory = new StarTreeDocValuesIteratorAdapter(); + starTreeDocValuesIteratorAdapter = new StarTreeDocValuesIteratorAdapter(); mockFieldInfo = new FieldInfo( "field", 1, @@ -59,38 +60,72 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); - assertEquals(iterator.getClass(), result.getClass()); + CoordinatedDocumentReader result = starTreeDocValuesIteratorAdapter.getDocValuesIterator( + DocValuesType.SORTED_NUMERIC, + mockFieldInfo, + producer + ); + assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); } public void testCreateIterator_UnsupportedType() { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { - factory.getDocValuesIterator(DocValuesType.BINARY, mockFieldInfo, producer); + starTreeDocValuesIteratorAdapter.getDocValuesIterator(DocValuesType.BINARY, mockFieldInfo, producer); }); assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - - long result = factory.getNextValue(iterator); + CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); + coordinatedDocumentReader.getDocIdSetIterator().nextDoc(); + long result = starTreeDocValuesIteratorAdapter.getNextValue(coordinatedDocumentReader, 0); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { + starTreeDocValuesIteratorAdapter.getNextValue(coordinatedDocumentReader, 0); + }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); } public void testNextDoc() throws IOException { - DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); when(iterator.nextDoc()).thenReturn(5); - int result = factory.nextDoc(iterator); + int result = starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader, 5); assertEquals(5, result); } + + public void test_multipleCoordinatedDocumentReader() throws IOException { + SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); + SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); + + CoordinatedDocumentReader coordinatedDocumentReader1 = new CoordinatedDocumentReader(iterator1); + CoordinatedDocumentReader coordinatedDocumentReader2 = new CoordinatedDocumentReader(iterator2); + + when(iterator1.nextDoc()).thenReturn(0); + when(iterator2.nextDoc()).thenReturn(1); + + when(iterator1.nextValue()).thenReturn(9L); + when(iterator2.nextValue()).thenReturn(9L); + + starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader1, 0); + starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader2, 0); + assertEquals(0, coordinatedDocumentReader1.getDocId()); + assertEquals(9L, (long) coordinatedDocumentReader1.getDocValue()); + assertNotEquals(0, coordinatedDocumentReader2.getDocId()); + assertEquals(1, coordinatedDocumentReader2.getDocId()); + assertEquals(9L, (long) coordinatedDocumentReader2.getDocValue()); + + } + } From 1b8c50f6956c9726c16ec319aa4f2a9faa2f8eb1 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Sun, 30 Jun 2024 22:16:37 +0530 Subject: [PATCH 06/16] addressing review comments Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 22 +++++---- .../aggregators/CountValueAggregator.java | 31 +++++------- .../aggregators/MetricAggregatorInfo.java | 8 ++-- .../aggregators/SumValueAggregator.java | 12 +++++ .../numerictype/StarTreeNumericType.java | 5 ++ .../StarTreeNumericTypeConverters.java | 6 +++ .../builder/OnHeapStarTreeBuilder.java | 6 +-- .../StarTreeDocValuesIteratorAdapter.java | 30 ++++++------ .../startree/builder/StarTreesBuilder.java | 8 ++-- ....java => SequentialDocValuesIterator.java} | 6 +-- .../CountValueAggregatorTests.java | 17 +++---- .../builder/BaseStarTreeBuilderTests.java | 2 +- .../builder/OnHeapStarTreeBuilderTests.java | 48 +++++++++---------- .../StarTreeValuesIteratorFactoryTests.java | 36 +++++++------- 14 files changed, 126 insertions(+), 111 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/{CoordinatedDocumentReader.java => SequentialDocValuesIterator.java} (91%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 728b630da3efd..309bba2b79ea0 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -22,7 +22,7 @@ import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; -import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -66,7 +66,7 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); - protected CoordinatedDocumentReader[] dimensionReaders; + protected SequentialDocValuesIterator[] dimensionReaders; protected Map fieldProducerMap; @@ -100,7 +100,7 @@ protected BaseStarTreeBuilder( this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = state.segmentInfo.maxDoc(); - this.dimensionReaders = new CoordinatedDocumentReader[numDimensions]; + this.dimensionReaders = new SequentialDocValuesIterator[numDimensions]; Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); for (int i = 0; i < numDimensions; i++) { @@ -133,7 +133,7 @@ public List generateMetricAggregatorInfos(MapperService ma for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { IndexNumericFieldData.NumericType numericType; - CoordinatedDocumentReader metricStatReader = null; + SequentialDocValuesIterator metricStatReader = null; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); @@ -206,7 +206,7 @@ public List generateMetricAggregatorInfos(MapperService ma * @param numDocs number of documents in the given segment * @return Iterator for the aggregated star-tree document */ - public abstract Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException; + public abstract Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException; /** * Generates aggregated star-tree documents for star-node. @@ -244,13 +244,15 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { starTreeDocValuesIteratorAdapter.nextDoc(dimensionReaders[i], currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); - throw e; + throw new RuntimeException("unable to iterate to next doc", e); } catch (Exception e) { logger.error("unable to read the dimension values from the segment", e); throw new IllegalStateException("unable to read the dimension values from the segment", e); } dimensions[i] = starTreeDocValuesIteratorAdapter.getNextValue(dimensionReaders[i], currentDocId); + } else { + throw new IllegalStateException("dimension readers are empty"); } } return dimensions; @@ -265,18 +267,20 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { private Object[] getStarTreeMetricsFromSegment(int currentDocId) throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - CoordinatedDocumentReader metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); + SequentialDocValuesIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { starTreeDocValuesIteratorAdapter.nextDoc(metricStatReader, currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); - throw e; + throw new RuntimeException("unable to iterate to next doc", e); } catch (Exception e) { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } metrics[i] = starTreeDocValuesIteratorAdapter.getNextValue(metricStatReader, currentDocId); + } else { + throw new IllegalStateException("metric readers are empty"); } } return metrics; @@ -403,7 +407,7 @@ public void build() throws IOException { long startTime = System.currentTimeMillis(); logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - Iterator starTreeDocumentIterator = sortAndAggregateStarTreeDocument(totalSegmentDocs); + Iterator starTreeDocumentIterator = sortAndAggregateStarTreeDocuments(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index 193c66cfa033c..26b99c1f17115 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; @@ -16,8 +15,8 @@ * * @opensearch.experimental */ -public class CountValueAggregator implements ValueAggregator { - public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; +public class CountValueAggregator implements ValueAggregator { + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; @Override public MetricStat getAggregationType() { @@ -30,22 +29,22 @@ public StarTreeNumericType getAggregatedValueType() { } @Override - public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { - return 1.0; + public Long getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return 1L; } @Override - public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Long mergeAggregatedValueAndSegmentValue(Long value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return value + 1; } @Override - public Double mergeAggregatedValues(Double value, Double aggregatedValue) { + public Long mergeAggregatedValues(Long value, Long aggregatedValue) { return value + aggregatedValue; } @Override - public Double getInitialAggregatedValue(Double value) { + public Long getInitialAggregatedValue(Long value) { return value; } @@ -55,20 +54,12 @@ public int getMaxAggregatedValueByteSize() { } @Override - public Long toLongValue(Double value) { - try { - return NumericUtils.doubleToSortableLong(value); - } catch (Exception e) { - throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); - } + public Long toLongValue(Long value) { + return value; } @Override - public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { - try { - return type.getDoubleValue(value); - } catch (Exception e) { - throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); - } + public Long toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { + return value; } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index d2ae3e11c9d7b..279e21b75ee18 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -9,7 +9,7 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.fielddata.IndexNumericFieldData; import java.util.Comparator; @@ -28,7 +28,7 @@ public class MetricAggregatorInfo implements Comparable { private final String field; private final ValueAggregator valueAggregators; private final StarTreeNumericType starTreeNumericType; - private final CoordinatedDocumentReader metricStatReader; + private final SequentialDocValuesIterator metricStatReader; /** * Constructor for MetricAggregatorInfo @@ -38,7 +38,7 @@ public MetricAggregatorInfo( String field, String starFieldName, IndexNumericFieldData.NumericType numericType, - CoordinatedDocumentReader metricStatReader + SequentialDocValuesIterator metricStatReader ) { this.metricStat = metricStat; this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat); @@ -87,7 +87,7 @@ public StarTreeNumericType getAggregatedValueType() { /** * @return metric value reader iterator */ - public CoordinatedDocumentReader getMetricStatReader() { + public SequentialDocValuesIterator getMetricStatReader() { return metricStatReader; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index 663b404e380db..543b0f7f42374 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -20,6 +20,8 @@ public class SumValueAggregator implements ValueAggregator { public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; + private double sum = 0; + private double compensation = 0; private CompensatedSum kahanSummation = new CompensatedSum(0, 0); @Override @@ -36,20 +38,28 @@ public StarTreeNumericType getAggregatedValueType() { public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { kahanSummation.reset(0, 0); kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + compensation = kahanSummation.delta(); + sum = kahanSummation.value(); return kahanSummation.value(); } @Override public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { assert kahanSummation.value() == value; + kahanSummation.reset(sum, compensation); kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); + compensation = kahanSummation.delta(); + sum = kahanSummation.value(); return kahanSummation.value(); } @Override public Double mergeAggregatedValues(Double value, Double aggregatedValue) { assert kahanSummation.value() == aggregatedValue; + kahanSummation.reset(sum, compensation); kahanSummation.add(value); + compensation = kahanSummation.delta(); + sum = kahanSummation.value(); return kahanSummation.value(); } @@ -57,6 +67,8 @@ public Double mergeAggregatedValues(Double value, Double aggregatedValue) { public Double getInitialAggregatedValue(Double value) { kahanSummation.reset(0, 0); kahanSummation.add(value); + compensation = kahanSummation.delta(); + sum = kahanSummation.value(); return kahanSummation.value(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index f09fb1567cb26..8cfb6821289a4 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -17,12 +17,15 @@ * @opensearch.experimental */ public enum StarTreeNumericType { + + // TODO: Handle scaled floats HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), + BYTE(IndexNumericFieldData.NumericType.BYTE, StarTreeNumericTypeConverters::bytesToDouble), UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); final IndexNumericFieldData.NumericType numericType; @@ -53,6 +56,8 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.SHORT; case UNSIGNED_LONG: return StarTreeNumericType.UNSIGNED_LONG; + case BYTE: + return StarTreeNumericType.BYTE; default: throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index b840d9436a26a..ff76256e4ff91 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -48,4 +48,10 @@ public static double unsignedlongToDouble(Long value) { return Numbers.unsignedLongToDouble(value); } + public static double bytesToDouble(Long value) { + byte[] bytes = new byte[8]; + NumericUtils.longToSortableBytes(value, bytes, 0); + return NumericUtils.sortableLongToDouble(NumericUtils.sortableBytesToLong(bytes, 0)); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index be9304db92173..8eda381d03082 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -71,12 +71,12 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int currentDocId = 0; currentDocId < numDocs; currentDocId++) { starTreeDocuments[currentDocId] = getSegmentStarTreeDocument(currentDocId); } - return sortAndAggregateStarTreeDocument(starTreeDocuments); + return sortAndAggregateStarTreeDocuments(starTreeDocuments); } /** @@ -85,7 +85,7 @@ public Iterator sortAndAggregateStarTreeDocument(int numDocs) * @return iterator for star-tree documents * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - public Iterator sortAndAggregateStarTreeDocument(StarTreeDocument[] starTreeDocuments) throws IOException { + public Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index 8a4bc0f7f0d59..cb0350bb110b0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -14,7 +14,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import java.io.IOException; @@ -29,11 +29,11 @@ public class StarTreeDocValuesIteratorAdapter { /** * Creates an iterator for the given doc values type and field using the doc values producer */ - public CoordinatedDocumentReader getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) + public SequentialDocValuesIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { case SORTED_NUMERIC: - return new CoordinatedDocumentReader(producer.getSortedNumeric(field)); + return new SequentialDocValuesIterator(producer.getSortedNumeric(field)); default: throw new IllegalArgumentException("Unsupported DocValuesType: " + type); } @@ -42,27 +42,27 @@ public CoordinatedDocumentReader getDocValuesIterator(DocValuesType type, FieldI /** * Returns the next value for the given iterator */ - public Long getNextValue(CoordinatedDocumentReader coordinatedDocumentReader, int currentDocId) throws IOException { - if (coordinatedDocumentReader.getDocIdSetIterator() instanceof SortedNumericDocValues) { - SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) coordinatedDocumentReader.getDocIdSetIterator(); - if (coordinatedDocumentReader.getDocId() < 0 || coordinatedDocumentReader.getDocId() == DocIdSetIterator.NO_MORE_DOCS) { + public Long getNextValue(SequentialDocValuesIterator sequentialDocValuesIterator, int currentDocId) throws IOException { + if (sequentialDocValuesIterator.getDocIdSetIterator() instanceof SortedNumericDocValues) { + SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) sequentialDocValuesIterator.getDocIdSetIterator(); + if (sequentialDocValuesIterator.getDocId() < 0 || sequentialDocValuesIterator.getDocId() == DocIdSetIterator.NO_MORE_DOCS) { throw new IllegalStateException("invalid doc id to fetch the next value"); } - if (coordinatedDocumentReader.getDocValue() == null) { - coordinatedDocumentReader.setDocValue(sortedNumericDocValues.nextValue()); - return coordinatedDocumentReader.getDocValue(); + if (sequentialDocValuesIterator.getDocValue() == null) { + sequentialDocValuesIterator.setDocValue(sortedNumericDocValues.nextValue()); + return sequentialDocValuesIterator.getDocValue(); } - if (coordinatedDocumentReader.getDocId() == currentDocId) { - Long nextValue = coordinatedDocumentReader.getDocValue(); - coordinatedDocumentReader.setDocValue(null); + if (sequentialDocValuesIterator.getDocId() == currentDocId) { + Long nextValue = sequentialDocValuesIterator.getDocValue(); + sequentialDocValuesIterator.setDocValue(null); return nextValue; } else { return null; } } else { - throw new IllegalStateException("Unsupported Iterator: " + coordinatedDocumentReader.getDocIdSetIterator().toString()); + throw new IllegalStateException("Unsupported Iterator: " + sequentialDocValuesIterator.getDocIdSetIterator().toString()); } } @@ -70,7 +70,7 @@ public Long getNextValue(CoordinatedDocumentReader coordinatedDocumentReader, in * Moves to the next doc in the iterator * Returns the doc id for the next document from the given iterator */ - public int nextDoc(CoordinatedDocumentReader iterator, int currentDocId) throws IOException { + public int nextDoc(SequentialDocValuesIterator iterator, int currentDocId) throws IOException { if (iterator.getDocValue() != null) { return iterator.getDocId(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 6665ea59c4060..282b5a07972d5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -61,19 +61,19 @@ public StarTreesBuilder( } this.starTreeFields = starTreeFields; - if (starTreeFields.isEmpty()) { - throw new IllegalArgumentException("Must provide star-tree field to build star trees"); - } this.fieldProducerMap = fieldProducerMap; this.state = segmentWriteState; this.mapperService = mapperService; - } /** * Builds the star-trees. */ public void build() throws IOException { + if (starTreeFields.isEmpty()) { + logger.debug("no star-tree fields found, returning from star-tree builder"); + return; + } long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); logger.debug("Starting building {} star-trees with star-tree fields", numStarTrees); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java similarity index 91% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 0026cb7163235..1fdbddcc56fa0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/CoordinatedDocumentReader.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -17,7 +17,7 @@ * @opensearch.experimental */ @ExperimentalApi -public class CoordinatedDocumentReader { +public class SequentialDocValuesIterator { /** * The doc id set iterator associated for each field. @@ -35,11 +35,11 @@ public class CoordinatedDocumentReader { private int docId; /** - * Constructs a new CoordinatedDocumentReader instance with the given DocIdSetIterator. + * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator. * * @param docIdSetIterator the DocIdSetIterator to be associated with this instance */ - public CoordinatedDocumentReader(DocIdSetIterator docIdSetIterator) { + public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { this.docIdSetIterator = docIdSetIterator; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java index e7d659ee177f3..e30e203406a6c 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; @@ -25,32 +24,30 @@ public void testGetAggregatedValueType() { } public void testGetInitialAggregatedValueForSegmentDocValue() { - assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0); + assertEquals(1L, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0); } public void testMergeAggregatedValueAndSegmentValue() { - assertEquals(3.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + assertEquals(3L, aggregator.mergeAggregatedValueAndSegmentValue(2L, 3L, StarTreeNumericType.LONG), 0.0); } public void testMergeAggregatedValues() { - assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); + assertEquals(5L, aggregator.mergeAggregatedValues(2L, 3L), 0.0); } public void testGetInitialAggregatedValue() { - assertEquals(3.0, aggregator.getInitialAggregatedValue(3.0), 0.0); + assertEquals(3L, aggregator.getInitialAggregatedValue(3L), 0.0); } public void testGetMaxAggregatedValueByteSize() { - assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + assertEquals(Long.BYTES, aggregator.getMaxAggregatedValueByteSize()); } public void testToLongValue() { - SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals(NumericUtils.doubleToSortableLong(3.0), aggregator.toLongValue(3.0), 0.0); + assertEquals(3L, aggregator.toLongValue(3L), 0.0); } public void testToStarTreeNumericTypeValue() { - SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); + assertEquals(3L, aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.LONG), 0.0); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 9bbcd1546f3cd..966595cb3c480 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -171,7 +171,7 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortAndAggregateStarTreeDocument(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException { return null; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index a13ad545f65af..3ae1116094528 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -157,7 +157,7 @@ public void setup() throws IOException { builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); } - public void test_sortAndAggregateStarTreeDocument() throws IOException { + public void test_sortAndAggregateStarTreeDocuments() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -169,8 +169,8 @@ public void test_sortAndAggregateStarTreeDocument() throws IOException { starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -182,7 +182,7 @@ public void test_sortAndAggregateStarTreeDocument() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -203,7 +203,7 @@ public void test_sortAndAggregateStarTreeDocument() throws IOException { } - public void test_sortAndAggregateStarTreeDocument_nullMetric() throws IOException { + public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -225,7 +225,7 @@ public void test_sortAndAggregateStarTreeDocument_nullMetric() throws IOExceptio segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); @@ -255,8 +255,8 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) + new StarTreeDocument(new Long[] { Long.MIN_VALUE, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, Long.MAX_VALUE }, new Object[] { 35.0, 34.0, 3L }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -268,7 +268,7 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -301,8 +301,8 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { Double.MAX_VALUE + 9, 14.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, Double.MIN_VALUE + 22, 3L }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -314,7 +314,7 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); int numOfAggregatedDocuments = 0; while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); @@ -395,7 +395,7 @@ public void test_build_halfFloatMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -444,7 +444,7 @@ public void test_build_floatMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -493,7 +493,7 @@ public void test_build_longMetrics() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); @@ -505,14 +505,14 @@ public void test_build_longMetrics() throws IOException { private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( - new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new Long[] { -1L, 4L, 3L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { -1L, 4L, -1L, 1L }, new Double[] { 35.0, 34.0, 3.0 }), - new StarTreeDocument(new Long[] { -1L, 4L, -1L, 4L }, new Double[] { 21.0, 14.0, 2.0 }), - new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Double[] { 56.0, 48.0, 5.0 }), - new StarTreeDocument(new Long[] { -1L, -1L, -1L, -1L }, new Double[] { 56.0, 48.0, 5.0 }) + new StarTreeDocument(new Long[] { 2L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }), + new StarTreeDocument(new Long[] { 3L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L }), + new StarTreeDocument(new Long[] { -1L, 4L, 2L, 1L }, new Object[] { 35.0, 34.0, 3L }), + new StarTreeDocument(new Long[] { -1L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, 1L }, new Object[] { 35.0, 34.0, 3L }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, 4L }, new Object[] { 21.0, 14.0, 2L }), + new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L }), + new StarTreeDocument(new Long[] { -1L, -1L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L }) ); return expectedStarTreeDocuments.iterator(); } @@ -536,7 +536,7 @@ public void test_build() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } - Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocument(segmentStarTreeDocuments); + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index 4e865c4580fc1..1aba67533d52e 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -16,7 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.index.compositeindex.datacube.startree.utils.CoordinatedDocumentReader; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -60,7 +60,7 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - CoordinatedDocumentReader result = starTreeDocValuesIteratorAdapter.getDocValuesIterator( + SequentialDocValuesIterator result = starTreeDocValuesIteratorAdapter.getDocValuesIterator( DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer @@ -80,28 +80,28 @@ public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); - coordinatedDocumentReader.getDocIdSetIterator().nextDoc(); - long result = starTreeDocValuesIteratorAdapter.getNextValue(coordinatedDocumentReader, 0); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + sequentialDocValuesIterator.getDocIdSetIterator().nextDoc(); + long result = starTreeDocValuesIteratorAdapter.getNextValue(sequentialDocValuesIterator, 0); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - starTreeDocValuesIteratorAdapter.getNextValue(coordinatedDocumentReader, 0); + starTreeDocValuesIteratorAdapter.getNextValue(sequentialDocValuesIterator, 0); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); } public void testNextDoc() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); - CoordinatedDocumentReader coordinatedDocumentReader = new CoordinatedDocumentReader(iterator); + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); when(iterator.nextDoc()).thenReturn(5); - int result = starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader, 5); + int result = starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator, 5); assertEquals(5, result); } @@ -109,8 +109,8 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); - CoordinatedDocumentReader coordinatedDocumentReader1 = new CoordinatedDocumentReader(iterator1); - CoordinatedDocumentReader coordinatedDocumentReader2 = new CoordinatedDocumentReader(iterator2); + SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1); + SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2); when(iterator1.nextDoc()).thenReturn(0); when(iterator2.nextDoc()).thenReturn(1); @@ -118,13 +118,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { when(iterator1.nextValue()).thenReturn(9L); when(iterator2.nextValue()).thenReturn(9L); - starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader1, 0); - starTreeDocValuesIteratorAdapter.nextDoc(coordinatedDocumentReader2, 0); - assertEquals(0, coordinatedDocumentReader1.getDocId()); - assertEquals(9L, (long) coordinatedDocumentReader1.getDocValue()); - assertNotEquals(0, coordinatedDocumentReader2.getDocId()); - assertEquals(1, coordinatedDocumentReader2.getDocId()); - assertEquals(9L, (long) coordinatedDocumentReader2.getDocValue()); + starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator1, 0); + starTreeDocValuesIteratorAdapter.nextDoc(sequentialDocValuesIterator2, 0); + assertEquals(0, sequentialDocValuesIterator1.getDocId()); + assertEquals(9L, (long) sequentialDocValuesIterator1.getDocValue()); + assertNotEquals(0, sequentialDocValuesIterator2.getDocId()); + assertEquals(1, sequentialDocValuesIterator2.getDocId()); + assertEquals(9L, (long) sequentialDocValuesIterator2.getDocValue()); } From eaa1c7e0cffb0f00867c0609f2615ea36acfe9ac Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 1 Jul 2024 15:51:12 +0530 Subject: [PATCH 07/16] rebasing with main Signed-off-by: Sarthak Aggarwal --- .../startree/builder/BaseStarTreeBuilderTests.java | 11 ++++++++--- .../startree/builder/OnHeapStarTreeBuilderTests.java | 8 +++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 966595cb3c480..10f4f41b27f6f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.BaseStarTreeBuilder; @@ -27,6 +26,7 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; @@ -80,7 +80,12 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { @BeforeClass public static void setup() throws IOException { - dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + dimensionsOrder = List.of( + new NumericDimension("field1"), + new NumericDimension("field3"), + new NumericDimension("field5"), + new NumericDimension("field8") + ); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); starTreeField = new StarTreeField( @@ -89,7 +94,7 @@ public static void setup() throws IOException { metrics, new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) ); - DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); directory = newFSDirectory(createTempDir()); SegmentInfo segmentInfo = new SegmentInfo( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 3ae1116094528..146fd97ce5ee4 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -27,6 +27,7 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; @@ -78,7 +79,12 @@ public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { @Before public void setup() throws IOException { - dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + dimensionsOrder = List.of( + new NumericDimension("field1"), + new NumericDimension("field3"), + new NumericDimension("field5"), + new NumericDimension("field8") + ); metrics = List.of( new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM)), From 8bf4a29a40eca6aef1971b0b06ed600bb90bd0fd Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 5 Jul 2024 10:10:32 +0530 Subject: [PATCH 08/16] support for empty sequential doc values iterator Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 6 ++-- .../utils/SequentialDocValuesIterator.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 309bba2b79ea0..95fbd40066a7d 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -133,7 +133,7 @@ public List generateMetricAggregatorInfos(MapperService ma for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { IndexNumericFieldData.NumericType numericType; - SequentialDocValuesIterator metricStatReader = null; + SequentialDocValuesIterator metricStatReader; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); @@ -145,12 +145,14 @@ public List generateMetricAggregatorInfos(MapperService ma FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); if (metricType != MetricStat.COUNT) { - // Need not initialize the metric reader for COUNT metric type + // Need not initialize the metric reader with relevant doc id set iterator for COUNT metric type metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( metricDocValuesType, metricFieldInfos, fieldProducerMap.get(metricFieldInfos.name) ); + } else { + metricStatReader = new SequentialDocValuesIterator(); } MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 1fdbddcc56fa0..b492b58fa7b89 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -14,6 +14,7 @@ /** * Coordinates the reading of documents across multiple DocIdSetIterators. * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. + * * @opensearch.experimental */ @ExperimentalApi @@ -43,6 +44,34 @@ public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { this.docIdSetIterator = docIdSetIterator; } + /** + * Creates a SequentialDocValuesIterator with an empty DocIdSetIterator. + * + */ + public SequentialDocValuesIterator() { + this.docIdSetIterator = new DocIdSetIterator() { + @Override + public int docID() { + return NO_MORE_DOCS; + } + + @Override + public int nextDoc() { + return NO_MORE_DOCS; + } + + @Override + public int advance(int target) { + return NO_MORE_DOCS; + } + + @Override + public long cost() { + return 0; + } + }; + } + /** * Returns the value associated with the latest document. * From d6841097abdcb5128d13573648a77db7bb595a40 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 5 Jul 2024 10:32:35 +0530 Subject: [PATCH 09/16] nits Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 23 ++++++--------- .../datacube/startree/StarTreeDocument.java | 1 + .../aggregators/MetricAggregatorInfo.java | 1 + .../numerictype/StarTreeNumericType.java | 1 + .../StarTreeNumericTypeConverters.java | 1 + .../aggregators/numerictype/package-info.java | 3 +- .../startree/aggregators/package-info.java | 1 + .../builder/OnHeapStarTreeBuilder.java | 11 ++++---- .../startree/builder/StarTreeBuilder.java | 1 + .../startree/builder/package-info.java | 1 + .../datacube/startree/package-info.java | 2 ++ .../utils/SequentialDocValuesIterator.java | 28 ------------------- .../startree/utils/StarTreeBuilderUtils.java | 1 + .../datacube/startree/utils/package-info.java | 1 + .../builder/BaseStarTreeBuilderTests.java | 4 +-- 15 files changed, 30 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 95fbd40066a7d..77935fd660fce 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -144,16 +144,11 @@ public List generateMetricAggregatorInfos(MapperService ma FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); - if (metricType != MetricStat.COUNT) { - // Need not initialize the metric reader with relevant doc id set iterator for COUNT metric type - metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( - metricDocValuesType, - metricFieldInfos, - fieldProducerMap.get(metricFieldInfos.name) - ); - } else { - metricStatReader = new SequentialDocValuesIterator(); - } + metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( + metricDocValuesType, + metricFieldInfos, + fieldProducerMap.get(metricFieldInfos.name) + ); MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( metricType, @@ -199,7 +194,7 @@ public List generateMetricAggregatorInfos(MapperService ma * @param dimensionId dimension id * @return dimension value */ - public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; + public abstract Long getDimensionValue(int docId, int dimensionId) throws IOException; /** * Sorts and aggregates the star-tree document in the segment, and returns a star-tree document iterator for all the @@ -522,10 +517,10 @@ private Map constructNonStarNodes(int start throws IOException { Map nodes = new HashMap<>(); int nodeStartDocId = startDocId; - long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); + Long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); for (int i = startDocId + 1; i < endDocId; i++) { - long dimensionValue = getDimensionValue(i, dimensionId); - if (dimensionValue != nodeDimensionValue) { + Long dimensionValue = getDimensionValue(i, dimensionId); + if (!dimensionValue.equals(nodeDimensionValue)) { StarTreeBuilderUtils.TreeNode child = getNewNode(); child.dimensionId = dimensionId; child.dimensionValue = nodeDimensionValue; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java index 0fc66b27e65a9..0ce2b3a5cdac5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java @@ -14,6 +14,7 @@ /** * Star tree document + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index 279e21b75ee18..46f1b1ac11063 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -17,6 +17,7 @@ /** * Builds aggregation function and doc values field pair to support various aggregations + * * @opensearch.experimental */ public class MetricAggregatorInfo implements Comparable { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index 8cfb6821289a4..57fe573a6a93c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -14,6 +14,7 @@ /** * Enum to map Star Tree Numeric Types to Lucene's Numeric Type + * * @opensearch.experimental */ public enum StarTreeNumericType { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index ff76256e4ff91..eb7647c4f9851 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -15,6 +15,7 @@ /** * Numeric converters used during aggregations of metric values + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java index b9a9bb8f1427d..fe5c2a7ceb254 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java @@ -8,6 +8,7 @@ /** * Numeric Types for Composite Index Star Tree - * @opensearch.experimental + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java index 27565ffded2cf..bddd6a46fbbe8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java @@ -8,6 +8,7 @@ /** * Aggregators for Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 8eda381d03082..8d0f1722084da 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -21,9 +21,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; /** * On heap single tree builder + * * @opensearch.experimental */ @ExperimentalApi @@ -64,9 +66,8 @@ public List getStarTreeDocuments() { return starTreeDocuments; } - // TODO: should this be just long? @Override - public long getDimensionValue(int docId, int dimensionId) throws IOException { + public Long getDimensionValue(int docId, int dimensionId) throws IOException { return starTreeDocuments.get(docId).dimensions[dimensionId]; } @@ -90,7 +91,7 @@ public Iterator sortAndAggregateStarTreeDocuments(StarTreeDocu // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = 0; i < numDimensions; i++) { - if (o1.dimensions[i] != o2.dimensions[i]) { + if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } @@ -154,7 +155,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = dimensionId + 1; i < numDimensions; i++) { - if (o1.dimensions[i] != o2.dimensions[i]) { + if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } @@ -167,7 +168,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start private boolean hasSameDimensions(StarTreeDocument starTreeDocument1, StarTreeDocument starTreeDocument2) { for (int i = dimensionId + 1; i < numDimensions; i++) { - if (starTreeDocument1.dimensions[i] != starTreeDocument2.dimensions[i]) { + if (!Objects.equals(starTreeDocument1.dimensions[i], starTreeDocument2.dimensions[i])) { return false; } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index ef542a1c848f2..20af1b3bc7935 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -15,6 +15,7 @@ /** * A star-tree builder that builds a single star-tree. + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java index 80eed545ef8a5..9c97b076371a3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java @@ -8,6 +8,7 @@ /** * Builders for Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java index 4f4e670478e2f..6d6cb420f4a9e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java @@ -7,5 +7,7 @@ */ /** * Core classes for handling star tree index. + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index b492b58fa7b89..51773c4068e6e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -44,34 +44,6 @@ public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { this.docIdSetIterator = docIdSetIterator; } - /** - * Creates a SequentialDocValuesIterator with an empty DocIdSetIterator. - * - */ - public SequentialDocValuesIterator() { - this.docIdSetIterator = new DocIdSetIterator() { - @Override - public int docID() { - return NO_MORE_DOCS; - } - - @Override - public int nextDoc() { - return NO_MORE_DOCS; - } - - @Override - public int advance(int target) { - return NO_MORE_DOCS; - } - - @Override - public long cost() { - return 0; - } - }; - } - /** * Returns the value associated with the latest document. * diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 96e6681d40a76..5552c9747d4fd 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -13,6 +13,7 @@ /** * Util class for building star tree + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java index 92930de98970d..c7e8b04d42178 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java @@ -8,6 +8,7 @@ /** * Utility to support Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.utils; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 10f4f41b27f6f..a042cefea879f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -171,8 +171,8 @@ public List getStarTreeDocuments() { } @Override - public long getDimensionValue(int docId, int dimensionId) throws IOException { - return 0; + public Long getDimensionValue(int docId, int dimensionId) throws IOException { + return 0L; } @Override From c79b6ba8bd3ad84e78c33ead0fa35eac12170872 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 12 Jul 2024 09:34:05 +0530 Subject: [PATCH 10/16] handling for count Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 18 +++++--- .../utils/SequentialDocValuesIterator.java | 46 +++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 77935fd660fce..785a6a413e6c7 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -131,7 +131,7 @@ public List generateMetricAggregatorInfos(MapperService ma throws IOException { List metricAggregatorInfos = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { - for (MetricStat metricType : metric.getMetrics()) { + for (MetricStat metricStat : metric.getMetrics()) { IndexNumericFieldData.NumericType numericType; SequentialDocValuesIterator metricStatReader; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); @@ -144,14 +144,18 @@ public List generateMetricAggregatorInfos(MapperService ma FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); - metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( - metricDocValuesType, - metricFieldInfos, - fieldProducerMap.get(metricFieldInfos.name) - ); + if (metricStat != MetricStat.COUNT) { + metricStatReader = starTreeDocValuesIteratorAdapter.getDocValuesIterator( + metricDocValuesType, + metricFieldInfos, + fieldProducerMap.get(metricFieldInfos.name) + ); + } else { + metricStatReader = new SequentialDocValuesIterator(); + } MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( - metricType, + metricStat, metric.getField(), starTreeField.getName(), numericType, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java index 51773c4068e6e..cf5f3e94c1ca6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java @@ -8,9 +8,12 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; +import java.io.IOException; + /** * Coordinates the reading of documents across multiple DocIdSetIterators. * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. @@ -44,6 +47,49 @@ public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { this.docIdSetIterator = docIdSetIterator; } + /** + * Constructs a new SequentialDocValuesIterator instance with the given SortedNumericDocValues. + * + */ + public SequentialDocValuesIterator() { + this.docIdSetIterator = new SortedNumericDocValues() { + @Override + public long nextValue() throws IOException { + return 0; + } + + @Override + public int docValueCount() { + return 0; + } + + @Override + public boolean advanceExact(int i) throws IOException { + return false; + } + + @Override + public int docID() { + return 0; + } + + @Override + public int nextDoc() throws IOException { + return 0; + } + + @Override + public int advance(int i) throws IOException { + return 0; + } + + @Override + public long cost() { + return 0; + } + }; + } + /** * Returns the value associated with the latest document. * From b085555c5d7cc39458306170a51bf36db49d70eb Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Fri, 12 Jul 2024 12:18:45 +0530 Subject: [PATCH 11/16] adding more tests Signed-off-by: Sarthak Aggarwal --- .../startree/builder/StarTreesBuilder.java | 2 +- ...StarTreeDocValuesIteratorAdapterTests.java | 139 ++++++++++++++++++ .../builder/StarTreesBuilderTests.java | 130 ++++++++++++++++ .../SequentialDocValuesIteratorTests.java | 46 ++++++ 4 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 282b5a07972d5..f9fd5fc91a9ad 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -93,7 +93,7 @@ public void close() throws IOException { } - private static StarTreeBuilder getSingleTreeBuilder( + static StarTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, SegmentWriteState state, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java new file mode 100644 index 0000000000000..9c2621401faa4 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapterTests.java @@ -0,0 +1,139 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Collections; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class StarTreeDocValuesIteratorAdapterTests extends OpenSearchTestCase { + + private StarTreeDocValuesIteratorAdapter adapter; + + @Override + public void setUp() throws Exception { + super.setUp(); + adapter = new StarTreeDocValuesIteratorAdapter(); + } + + public void testGetDocValuesIterator() throws IOException { + DocValuesProducer mockProducer = mock(DocValuesProducer.class); + SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class); + + when(mockProducer.getSortedNumeric(any())).thenReturn(mockSortedNumericDocValues); + + SequentialDocValuesIterator iterator = adapter.getDocValuesIterator(DocValuesType.SORTED_NUMERIC, any(), mockProducer); + + assertNotNull(iterator); + assertEquals(mockSortedNumericDocValues, iterator.getDocIdSetIterator()); + } + + public void testGetDocValuesIteratorWithUnsupportedType() { + DocValuesProducer mockProducer = mock(DocValuesProducer.class); + FieldInfo fieldInfo = new FieldInfo( + "random_field", + 0, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + adapter.getDocValuesIterator(DocValuesType.BINARY, fieldInfo, mockProducer); + }); + + assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); + } + + public void testGetNextValue() throws IOException { + SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class); + SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues); + iterator.setDocId(1); + when(mockSortedNumericDocValues.nextValue()).thenReturn(42L); + + Long nextValue = adapter.getNextValue(iterator, 1); + + assertEquals(Long.valueOf(42L), nextValue); + assertEquals(Long.valueOf(42L), iterator.getDocValue()); + } + + public void testGetNextValueWithInvalidDocId() { + SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class); + SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues); + iterator.setDocId(DocIdSetIterator.NO_MORE_DOCS); + + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { adapter.getNextValue(iterator, 1); }); + + assertEquals("invalid doc id to fetch the next value", exception.getMessage()); + } + + public void testGetNextValueWithUnsupportedIterator() { + DocIdSetIterator mockIterator = mock(DocIdSetIterator.class); + SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockIterator); + + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { adapter.getNextValue(iterator, 1); }); + + assertEquals("Unsupported Iterator: " + mockIterator.toString(), exception.getMessage()); + } + + public void testNextDoc() throws IOException { + SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class); + SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues); + when(mockSortedNumericDocValues.nextDoc()).thenReturn(2, 3, DocIdSetIterator.NO_MORE_DOCS); + when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L); + + int nextDocId = adapter.nextDoc(iterator, 1); + assertEquals(2, nextDocId); + assertEquals(Long.valueOf(42L), adapter.getNextValue(iterator, nextDocId)); + + nextDocId = adapter.nextDoc(iterator, 2); + assertEquals(3, nextDocId); + when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L); + + } + + public void testNextDoc_noMoreDocs() throws IOException { + SortedNumericDocValues mockSortedNumericDocValues = mock(SortedNumericDocValues.class); + SequentialDocValuesIterator iterator = new SequentialDocValuesIterator(mockSortedNumericDocValues); + when(mockSortedNumericDocValues.nextDoc()).thenReturn(2, DocIdSetIterator.NO_MORE_DOCS); + when(mockSortedNumericDocValues.nextValue()).thenReturn(42L, 32L); + + int nextDocId = adapter.nextDoc(iterator, 1); + assertEquals(2, nextDocId); + assertEquals(Long.valueOf(42L), adapter.getNextValue(iterator, nextDocId)); + + assertThrows(IllegalStateException.class, () -> adapter.nextDoc(iterator, 2)); + + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java new file mode 100644 index 0000000000000..88003c183f98b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java @@ -0,0 +1,130 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.StarTreeMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +public class StarTreesBuilderTests extends OpenSearchTestCase { + + private MapperService mapperService; + private SegmentWriteState segmentWriteState; + private DocValuesProducer docValuesProducer; + private StarTreeMapper.StarTreeFieldType starTreeFieldType; + private StarTreeField starTreeField; + private Map fieldProducerMap; + private Directory directory; + + public void setUp() throws Exception { + super.setUp(); + mapperService = mock(MapperService.class); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]); + segmentWriteState = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + docValuesProducer = mock(DocValuesProducer.class); + StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration( + 1, + new HashSet<>(), + StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP + ); + starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); + starTreeFieldType = new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField); + fieldProducerMap = new HashMap<>(); + fieldProducerMap.put("field1", docValuesProducer); + } + + public void test_buildWithNoStarTreeFields() throws IOException { + when(mapperService.getCompositeFieldTypes()).thenReturn(new HashSet<>()); + + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService); + starTreesBuilder.build(); + + verifyNoInteractions(docValuesProducer); + } + + public void test_getSingleTreeBuilder() throws IOException { + when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); + StarTreeBuilder starTreeBuilder = StarTreesBuilder.getSingleTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService); + assertTrue(starTreeBuilder instanceof OnHeapStarTreeBuilder); + } + + public void test_getSingleTreeBuilder_illegalArgument() { + when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); + StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP); + StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); + assertThrows(IllegalArgumentException.class, () -> StarTreesBuilder.getSingleTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService)); + } + + public void test_closeWithNoStarTreeFields() throws IOException { + StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration( + 1, + new HashSet<>(), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); + starTreeFieldType = new StarTreeMapper.StarTreeFieldType("star_tree", starTreeField); + when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService); + starTreesBuilder.close(); + + verifyNoInteractions(docValuesProducer); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java new file mode 100644 index 0000000000000..76b612e3677f7 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.index.fielddata.AbstractNumericDocValues; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class SequentialDocValuesIteratorTests extends OpenSearchTestCase { + + public void test_sequentialDocValuesIterator() { + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(new AbstractNumericDocValues() { + @Override + public long longValue() throws IOException { + return 0; + } + + @Override + public boolean advanceExact(int i) throws IOException { + return false; + } + + @Override + public int docID() { + return 0; + } + }); + + assertTrue(sequentialDocValuesIterator.getDocIdSetIterator() instanceof AbstractNumericDocValues); + assertEquals(sequentialDocValuesIterator.getDocId(), 0); + } + + public void test_sequentialDocValuesIterator_default() { + SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(); + assertTrue(sequentialDocValuesIterator.getDocIdSetIterator() instanceof SortedNumericDocValues); + } + +} From 027daf698dd7809716c5076ecfaa91a65ac54a24 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 16 Jul 2024 07:34:00 +0530 Subject: [PATCH 12/16] addressing review comments Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 45 ++++++------ .../composite/Composite99DocValuesWriter.java | 5 +- .../aggregators/CountValueAggregator.java | 3 +- .../builder/OnHeapStarTreeBuilder.java | 2 +- .../startree/utils/StarTreeBuilderUtils.java | 73 ------------------- .../datacube/startree/utils/TreeNode.java | 65 +++++++++++++++++ 6 files changed, 95 insertions(+), 98 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 785a6a413e6c7..69786ec60cb8b 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -23,7 +23,7 @@ import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; @@ -39,6 +39,8 @@ import java.util.Map; import java.util.Set; +import static org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode.ALL; + /** * Builder for star tree. Defines the algorithm to construct star-tree * See {@link StarTreesBuilder} for information around the construction of star-trees based on star-tree fields @@ -64,7 +66,7 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected int numStarTreeNodes; protected final int maxLeafDocuments; - protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); + protected final TreeNode rootNode = getNewNode(); protected SequentialDocValuesIterator[] dimensionReaders; @@ -237,7 +239,7 @@ protected StarTreeDocument getSegmentStarTreeDocument(int currentDocId) throws I * @return dimension values for each of the star-tree dimension * @throws IOException when we are unable to iterate to the next doc for the given dimension readers */ - Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { + private Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { Long[] dimensions = new Long[numDimensions]; for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { @@ -470,9 +472,9 @@ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOExcept * * @return return new star-tree node */ - private StarTreeBuilderUtils.TreeNode getNewNode() { + private TreeNode getNewNode() { numStarTreeNodes++; - return new StarTreeBuilderUtils.TreeNode(); + return new TreeNode(); } /** @@ -483,7 +485,7 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ - private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { + private void constructStarTree(TreeNode node, int startDocId, int endDocId) throws IOException { int childDimensionId = node.dimensionId + 1; if (childDimensionId == numDimensions) { @@ -492,16 +494,16 @@ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocI // Construct all non-star children nodes node.childDimensionId = childDimensionId; - Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); + Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); node.children = children; // Construct star-node if required if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) { - children.put((long) StarTreeBuilderUtils.ALL, constructStarNode(startDocId, endDocId, childDimensionId)); + children.put((long) ALL, constructStarNode(startDocId, endDocId, childDimensionId)); } // Further split on child nodes if required - for (StarTreeBuilderUtils.TreeNode child : children.values()) { + for (TreeNode child : children.values()) { if (child.endDocId - child.startDocId > maxLeafDocuments) { constructStarTree(child, child.startDocId, child.endDocId); } @@ -517,15 +519,14 @@ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocI * @return root node with non-star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes */ - private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) - throws IOException { - Map nodes = new HashMap<>(); + private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) throws IOException { + Map nodes = new HashMap<>(); int nodeStartDocId = startDocId; Long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); for (int i = startDocId + 1; i < endDocId; i++) { Long dimensionValue = getDimensionValue(i, dimensionId); if (!dimensionValue.equals(nodeDimensionValue)) { - StarTreeBuilderUtils.TreeNode child = getNewNode(); + TreeNode child = getNewNode(); child.dimensionId = dimensionId; child.dimensionValue = nodeDimensionValue; child.startDocId = nodeStartDocId; @@ -536,7 +537,7 @@ private Map constructNonStarNodes(int start nodeDimensionValue = dimensionValue; } } - StarTreeBuilderUtils.TreeNode lastNode = getNewNode(); + TreeNode lastNode = getNewNode(); lastNode.dimensionId = dimensionId; lastNode.dimensionValue = nodeDimensionValue; lastNode.startDocId = nodeStartDocId; @@ -554,10 +555,10 @@ private Map constructNonStarNodes(int start * @return root node with star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes */ - private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { - StarTreeBuilderUtils.TreeNode starNode = getNewNode(); + private TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + TreeNode starNode = getNewNode(); starNode.dimensionId = dimensionId; - starNode.dimensionValue = StarTreeBuilderUtils.ALL; + starNode.dimensionValue = ALL; starNode.isStarNode = true; starNode.startDocId = numStarTreeDocs; Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); @@ -575,11 +576,11 @@ private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endD * @return aggregated star-tree documents * @throws IOException throws an exception upon failing to create new aggregated docs based on star tree */ - private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node) throws IOException { + private StarTreeDocument createAggregatedDocs(TreeNode node) throws IOException { StarTreeDocument aggregatedStarTreeDocument = null; if (node.children == null) { - // For leaf node + // For leaf node if (node.startDocId == node.endDocId - 1) { // If it has only one document, use it as the aggregated document aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId); @@ -600,9 +601,9 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } } else { // For non-leaf node - if (node.children.containsKey((long) StarTreeBuilderUtils.ALL)) { + if (node.children.containsKey((long) ALL)) { // If it has star child, use the star child aggregated document directly - for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + for (TreeNode child : node.children.values()) { if (child.isStarNode) { aggregatedStarTreeDocument = createAggregatedDocs(child); node.aggregatedDocId = child.aggregatedDocId; @@ -612,7 +613,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } } else { // If no star child exists, aggregate all aggregated documents from non-star children - for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + for (TreeNode child : node.children.values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } if (null == aggregatedStarTreeDocument) { diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java index 75bbf78dbdad2..62da9fde6d831 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; @@ -98,7 +99,9 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, if (compositeFieldSet.isEmpty()) { for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { - // TODO : Call StarTree builder + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, state, mapperService); + starTreesBuilder.build(); + starTreesBuilder.close(); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index 26b99c1f17115..d72f4a292dc0a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -17,6 +17,7 @@ */ public class CountValueAggregator implements ValueAggregator { public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.LONG; + public static final long DEFAULT_INITIAL_VALUE = 1L; @Override public MetricStat getAggregationType() { @@ -30,7 +31,7 @@ public StarTreeNumericType getAggregatedValueType() { @Override public Long getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { - return 1L; + return DEFAULT_INITIAL_VALUE; } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 8d0f1722084da..489004884acd3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -86,7 +86,7 @@ public Iterator sortAndAggregateStarTreeDocuments(int numDocs) * @return iterator for star-tree documents * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - public Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java deleted file mode 100644 index 5552c9747d4fd..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.utils; - -import org.opensearch.common.annotation.ExperimentalApi; - -import java.util.Map; - -/** - * Util class for building star tree - * - * @opensearch.experimental - */ -@ExperimentalApi -public class StarTreeBuilderUtils { - - private StarTreeBuilderUtils() {} - - public static final int ALL = -1; - - /** - * Represents a node in a tree data structure, specifically designed for a star-tree implementation. - * A star-tree node will represent both star and non-star nodes. - */ - public static class TreeNode { - - /** - * The dimension id for the dimension (field) associated with this star-tree node. - */ - public int dimensionId = ALL; - - /** - * The starting document id (inclusive) associated with this star-tree node. - */ - public int startDocId = ALL; - - /** - * The ending document id (exclusive) associated with this star-tree node. - */ - public int endDocId = ALL; - - /** - * The aggregated document id associated with this star-tree node. - */ - public int aggregatedDocId = ALL; - - /** - * The child dimension identifier associated with this star-tree node. - */ - public int childDimensionId = ALL; - - /** - * The value of the dimension associated with this star-tree node. - */ - public long dimensionValue = ALL; - - /** - * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions). - */ - public boolean isStarNode = false; - - /** - * A map containing the child nodes of this star-tree node, keyed by their dimension id. - */ - public Map children; - } - -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java new file mode 100644 index 0000000000000..5cf737c61ab2d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/TreeNode.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Map; + +/** + * /** + * Represents a node in a tree data structure, specifically designed for a star-tree implementation. + * A star-tree node will represent both star and non-star nodes. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class TreeNode { + + public static final int ALL = -1; + + /** + * The dimension id for the dimension (field) associated with this star-tree node. + */ + public int dimensionId = ALL; + + /** + * The starting document id (inclusive) associated with this star-tree node. + */ + public int startDocId = ALL; + + /** + * The ending document id (exclusive) associated with this star-tree node. + */ + public int endDocId = ALL; + + /** + * The aggregated document id associated with this star-tree node. + */ + public int aggregatedDocId = ALL; + + /** + * The child dimension identifier associated with this star-tree node. + */ + public int childDimensionId = ALL; + + /** + * The value of the dimension associated with this star-tree node. + */ + public long dimensionValue = ALL; + + /** + * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions). + */ + public boolean isStarNode = false; + + /** + * A map containing the child nodes of this star-tree node, keyed by their dimension id. + */ + public Map children; +} From 3b6cfcac562fc6fd6702b0b94b2d28c4f11ffd0c Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 16 Jul 2024 07:45:08 +0530 Subject: [PATCH 13/16] addressed nits Signed-off-by: Sarthak Aggarwal --- .../composite/Composite99DocValuesWriter.java | 6 +- .../builder}/BaseStarTreeBuilder.java | 69 +++++++++++-------- .../builder/OnHeapStarTreeBuilder.java | 60 +++++++++------- .../startree/builder/StarTreesBuilder.java | 7 +- .../StarTreeDocValuesFormatTests.java | 36 ---------- .../builder/BaseStarTreeBuilderTests.java | 3 +- .../builder/StarTreesBuilderTests.java | 10 +-- 7 files changed, 90 insertions(+), 101 deletions(-) rename server/src/main/java/org/{apache/lucene/index => opensearch/index/compositeindex/datacube/startree/builder}/BaseStarTreeBuilder.java (93%) diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java index 62da9fde6d831..3753b20a8bea3 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java @@ -99,9 +99,9 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, if (compositeFieldSet.isEmpty()) { for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) { if (mappedType instanceof StarTreeMapper.StarTreeFieldType) { - StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, state, mapperService); - starTreesBuilder.build(); - starTreesBuilder.close(); + try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, state, mapperService)) { + starTreesBuilder.build(); + } } } } diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java similarity index 93% rename from server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 69786ec60cb8b..0333db7256994 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -5,11 +5,14 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.apache.lucene.index; +package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -19,9 +22,6 @@ import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; -import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; -import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.TreeNode; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -70,6 +70,7 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { protected SequentialDocValuesIterator[] dimensionReaders; + // We do not close these producers as they are empty doc value producers (where close() is unsupported) protected Map fieldProducerMap; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorAdapter; @@ -206,10 +207,9 @@ public List generateMetricAggregatorInfos(MapperService ma * Sorts and aggregates the star-tree document in the segment, and returns a star-tree document iterator for all the * aggregated star-tree document. * - * @param numDocs number of documents in the given segment * @return Iterator for the aggregated star-tree document */ - public abstract Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException; + public abstract Iterator sortAndAggregateStarTreeDocuments() throws IOException; /** * Generates aggregated star-tree documents for star-node. @@ -242,25 +242,35 @@ protected StarTreeDocument getSegmentStarTreeDocument(int currentDocId) throws I private Long[] getStarTreeDimensionsFromSegment(int currentDocId) throws IOException { Long[] dimensions = new Long[numDimensions]; for (int i = 0; i < numDimensions; i++) { - if (dimensionReaders[i] != null) { - try { - starTreeDocValuesIteratorAdapter.nextDoc(dimensionReaders[i], currentDocId); - } catch (IOException e) { - logger.error("unable to iterate to next doc", e); - throw new RuntimeException("unable to iterate to next doc", e); - } catch (Exception e) { - logger.error("unable to read the dimension values from the segment", e); - throw new IllegalStateException("unable to read the dimension values from the segment", e); - } - - dimensions[i] = starTreeDocValuesIteratorAdapter.getNextValue(dimensionReaders[i], currentDocId); - } else { - throw new IllegalStateException("dimension readers are empty"); + try { + dimensions[i] = getValuesFromSegment(dimensionReaders[i], currentDocId); + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); } + } return dimensions; } + /** + * Returns the next value from the iterator of respective field + * + * @param iterator respective field iterator + * @param currentDocId current document id + * @return the next value for the field + * @throws IOException when we are unable to iterate to the next doc for the given iterator + */ + private Long getValuesFromSegment(SequentialDocValuesIterator iterator, int currentDocId) throws IOException { + try { + starTreeDocValuesIteratorAdapter.nextDoc(iterator, currentDocId); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + throw new RuntimeException("unable to iterate to next doc", e); + } + return starTreeDocValuesIteratorAdapter.getNextValue(iterator, currentDocId); + } + /** * Returns the metric values for the next document from the segment * @@ -273,15 +283,11 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId) throws IOExcept SequentialDocValuesIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { - starTreeDocValuesIteratorAdapter.nextDoc(metricStatReader, currentDocId); - } catch (IOException e) { - logger.error("unable to iterate to next doc", e); - throw new RuntimeException("unable to iterate to next doc", e); + metrics[i] = getValuesFromSegment(metricStatReader, currentDocId); } catch (Exception e) { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorAdapter.getNextValue(metricStatReader, currentDocId); } else { throw new IllegalStateException("metric readers are empty"); } @@ -297,6 +303,7 @@ private Object[] getStarTreeMetricsFromSegment(int currentDocId) throws IOExcept * @param segmentDocument segment star-tree document * @return merged star-tree document */ + @SuppressWarnings({ "unchecked", "rawtypes" }) protected StarTreeDocument reduceSegmentStarTreeDocuments( StarTreeDocument aggregatedSegmentDocument, StarTreeDocument segmentDocument @@ -370,6 +377,7 @@ private static long getLong(Object metric) { * @param starTreeDocument segment star-tree document * @return merged star-tree document */ + @SuppressWarnings("unchecked") public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocument, StarTreeDocument starTreeDocument) { // aggregate the documents if (aggregatedDocument == null) { @@ -410,7 +418,12 @@ public void build() throws IOException { long startTime = System.currentTimeMillis(); logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - Iterator starTreeDocumentIterator = sortAndAggregateStarTreeDocuments(totalSegmentDocs); + if (totalSegmentDocs == 0) { + logger.debug("No documents found in the segment"); + return; + } + + Iterator starTreeDocumentIterator = sortAndAggregateStarTreeDocuments(); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -422,7 +435,7 @@ public void build() throws IOException { * @param starTreeDocumentIterator contains the sorted and aggregated documents * @throws IOException when we are unable to build star-tree */ - public void build(Iterator starTreeDocumentIterator) throws IOException { + void build(Iterator starTreeDocumentIterator) throws IOException { int numSegmentStarTreeDocument = totalSegmentDocs; while (starTreeDocumentIterator.hasNext()) { @@ -432,7 +445,7 @@ public void build(Iterator starTreeDocumentIterator) throws IO logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); if (numStarTreeDocs == 0) { - // TODO: Uncomment when segment codec is ready + // TODO: Uncomment when segment codec and file formats is ready // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); return; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 489004884acd3..caeb24838da62 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; @@ -72,31 +71,26 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocuments() throws IOException { + int numDocs = totalSegmentDocs; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int currentDocId = 0; currentDocId < numDocs; currentDocId++) { starTreeDocuments[currentDocId] = getSegmentStarTreeDocument(currentDocId); } + return sortAndAggregateStarTreeDocuments(starTreeDocuments); } /** * Sort, aggregates and merges the star-tree documents + * * @param starTreeDocuments star-tree documents * @return iterator for star-tree documents - * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] starTreeDocuments) { - // sort the documents - Arrays.sort(starTreeDocuments, (o1, o2) -> { - for (int i = 0; i < numDimensions; i++) { - if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { - return Long.compare(o1.dimensions[i], o2.dimensions[i]); - } - } - return 0; - }); + // sort all the documents + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, 0); // merge the documents return mergeStarTreeDocuments(starTreeDocuments); @@ -104,6 +98,7 @@ Iterator sortAndAggregateStarTreeDocuments(StarTreeDocument[] /** * Merges the star-tree documents + * * @param starTreeDocuments star-tree documents * @return iterator to aggregate star-tree documents */ @@ -111,6 +106,7 @@ private Iterator mergeStarTreeDocuments(StarTreeDocument[] sta return new Iterator<>() { boolean hasNext = true; StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; + // starting from 1 since we have already fetched the 0th document int docId = 1; @Override @@ -123,8 +119,9 @@ public StarTreeDocument next() { // aggregate as we move on to the next doc StarTreeDocument next = reduceSegmentStarTreeDocuments(null, currentStarTreeDocument); while (docId < starTreeDocuments.length) { - StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; - if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { + StarTreeDocument starTreeDocument = starTreeDocuments[docId]; + docId++; + if (Arrays.equals(starTreeDocument.dimensions, next.dimensions) == false) { currentStarTreeDocument = starTreeDocument; return next; } else { @@ -139,6 +136,7 @@ public StarTreeDocument next() { /** * Generates a star-tree for a given star-node + * * @param startDocId Start document id in the star-tree * @param endDocId End document id (exclusive) in the star-tree * @param dimensionId Dimension id of the star-node @@ -153,14 +151,10 @@ public Iterator generateStarTreeDocumentsForStarNode(int start for (int i = 0; i < numDocs; i++) { starTreeDocuments[i] = getStarTreeDocument(startDocId + i); } - Arrays.sort(starTreeDocuments, (o1, o2) -> { - for (int i = dimensionId + 1; i < numDimensions; i++) { - if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { - return Long.compare(o1.dimensions[i], o2.dimensions[i]); - } - } - return 0; - }); + + // sort star tree documents from given dimension id (as previous dimension ids have already been processed) + sortStarTreeDocumentsFromDimensionId(starTreeDocuments, dimensionId + 1); + return new Iterator() { boolean hasNext = true; StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; @@ -185,7 +179,8 @@ public StarTreeDocument next() { StarTreeDocument next = reduceStarTreeDocuments(null, currentStarTreeDocument); next.dimensions[dimensionId] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); while (docId < numDocs) { - StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; + StarTreeDocument starTreeDocument = starTreeDocuments[docId]; + docId++; if (!hasSameDimensions(starTreeDocument, currentStarTreeDocument)) { currentStarTreeDocument = starTreeDocument; return next; @@ -198,4 +193,21 @@ public StarTreeDocument next() { } }; } + + /** + * Sorts the star-tree documents from the given dimension id + * + * @param starTreeDocuments star-tree documents + * @param dimensionId id of the dimension + */ + private void sortStarTreeDocumentsFromDimensionId(StarTreeDocument[] starTreeDocuments, int dimensionId) { + Arrays.sort(starTreeDocuments, (o1, o2) -> { + for (int i = dimensionId; i < numDimensions; i++) { + if (!Objects.equals(o1.dimensions[i], o2.dimensions[i])) { + return Long.compare(o1.dimensions[i], o2.dimensions[i]); + } + } + return 0; + }); + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index f9fd5fc91a9ad..eaf9ae1dcdaa1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -79,9 +79,8 @@ public void build() throws IOException { logger.debug("Starting building {} star-trees with star-tree fields", numStarTrees); // Build all star-trees - for (int i = 0; i < numStarTrees; i++) { - StarTreeField starTreeField = starTreeFields.get(i); - try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(starTreeField, fieldProducerMap, state, mapperService)) { + for (StarTreeField starTreeField : starTreeFields) { + try (StarTreeBuilder starTreeBuilder = getStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService)) { starTreeBuilder.build(); } } @@ -93,7 +92,7 @@ public void close() throws IOException { } - static StarTreeBuilder getSingleTreeBuilder( + StarTreeBuilder getStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, SegmentWriteState state, diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java index 6c6d26656e4de..31df9a49bebfb 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java @@ -12,12 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.Rounding; import org.opensearch.index.codec.composite.Composite99Codec; @@ -31,7 +26,6 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.StarTreeMapper; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -77,34 +71,4 @@ private static StarTreeField getStarTreeField(List d1Cale return new StarTreeField("starTree", dims, metrics, config); } - - public void testStarTreeDocValues() throws IOException { - Directory directory = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(null); - conf.setMergePolicy(newLogMergePolicy()); - RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); - Document doc = new Document(); - doc.add(new SortedNumericDocValuesField("sndv", 1)); - doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", 1)); - iw.addDocument(doc); - doc.add(new SortedNumericDocValuesField("sndv", 1)); - doc.add(new SortedNumericDocValuesField("dv", 1)); - doc.add(new SortedNumericDocValuesField("field", 1)); - iw.addDocument(doc); - iw.forceMerge(1); - doc.add(new SortedNumericDocValuesField("sndv", 2)); - doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", 2)); - iw.addDocument(doc); - doc.add(new SortedNumericDocValuesField("sndv", 2)); - doc.add(new SortedNumericDocValuesField("dv", 2)); - doc.add(new SortedNumericDocValuesField("field", 2)); - iw.addDocument(doc); - iw.forceMerge(1); - iw.close(); - - // TODO : validate star tree structures that got created - directory.close(); - } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index a042cefea879f..b78130e72aba1 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -176,7 +175,7 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator sortAndAggregateStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortAndAggregateStarTreeDocuments() throws IOException { return null; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java index 88003c183f98b..518c6729c2e1a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java @@ -94,17 +94,19 @@ public void test_buildWithNoStarTreeFields() throws IOException { verifyNoInteractions(docValuesProducer); } - public void test_getSingleTreeBuilder() throws IOException { + public void test_getStarTreeBuilder() throws IOException { when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); - StarTreeBuilder starTreeBuilder = StarTreesBuilder.getSingleTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService); + StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService); assertTrue(starTreeBuilder instanceof OnHeapStarTreeBuilder); } - public void test_getSingleTreeBuilder_illegalArgument() { + public void test_getStarTreeBuilder_illegalArgument() { when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType)); StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP); StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration); - assertThrows(IllegalArgumentException.class, () -> StarTreesBuilder.getSingleTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService)); + StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, segmentWriteState, mapperService); + assertThrows(IllegalArgumentException.class, () -> starTreesBuilder.getStarTreeBuilder(starTreeField, fieldProducerMap, segmentWriteState, mapperService)); } public void test_closeWithNoStarTreeFields() throws IOException { From 4943d9dc6dd38ce2111e9be7a32b6f65a28acac0 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 16 Jul 2024 13:33:59 +0530 Subject: [PATCH 14/16] add changelog Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b863b9d13e789..452579c6d3a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604)) - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750)) - Add Plugin interface for loading application based configuration templates (([#14659](https://github.com/opensearch-project/OpenSearch/issues/14659))) +- Star Tree Implementation (OnHeap) ([#14512](https://github.com/opensearch-project/OpenSearch/pull/14512)) ### Dependencies - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442)) From 7caf5a0904989a91867931f771ebb5dd83e2bda0 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 16 Jul 2024 23:02:48 +0530 Subject: [PATCH 15/16] added a test Signed-off-by: Sarthak Aggarwal --- .../startree/builder/BaseStarTreeBuilder.java | 27 +-- .../builder/OnHeapStarTreeBuilderTests.java | 166 +++++++++++++++--- 2 files changed, 163 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index 0333db7256994..c6249a87e7c00 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -626,17 +626,24 @@ private StarTreeDocument createAggregatedDocs(TreeNode node) throws IOException } } else { // If no star child exists, aggregate all aggregated documents from non-star children - for (TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); - } - if (null == aggregatedStarTreeDocument) { - throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); - } - for (int i = node.dimensionId + 1; i < numDimensions; i++) { - aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); + if (node.children.values().size() == 1) { + for (TreeNode child : node.children.values()) { + aggregatedStarTreeDocument = createAggregatedDocs(child); + node.aggregatedDocId = child.aggregatedDocId; + } + } else { + for (TreeNode child : node.children.values()) { + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); + } + if (null == aggregatedStarTreeDocument) { + throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); + } + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument.dimensions[i] = Long.valueOf(STAR_IN_DOC_VALUES_INDEX); + } + node.aggregatedDocId = numStarTreeDocs; + appendToStarTree(aggregatedStarTreeDocument); } - node.aggregatedDocId = numStarTreeDocs; - appendToStarTree(aggregatedStarTreeDocument); } } return aggregatedStarTreeDocument; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 146fd97ce5ee4..941fb4abfbe21 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -58,18 +58,7 @@ public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { private OnHeapStarTreeBuilder builder; private MapperService mapperService; private List dimensionsOrder; - private List fields = List.of( - "field1", - "field2", - "field3", - "field4", - "field5", - "field6", - "field7", - "field8", - "field9", - "field10" - ); + private List fields = List.of(); private List metrics; private Directory directory; private FieldInfo[] fieldsInfo; @@ -79,6 +68,8 @@ public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { @Before public void setup() throws IOException { + fields = List.of("field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", "field9", "field10"); + dimensionsOrder = List.of( new NumericDimension("field1"), new NumericDimension("field3"), @@ -295,7 +286,7 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() } - public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { + public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -405,7 +396,7 @@ public void test_build_halfFloatMetrics() throws IOException { builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(8, resultStarTreeDocuments.size()); + assertEquals(7, resultStarTreeDocuments.size()); Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); @@ -454,7 +445,7 @@ public void test_build_floatMetrics() throws IOException { builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(8, resultStarTreeDocuments.size()); + assertEquals(7, resultStarTreeDocuments.size()); Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); @@ -503,7 +494,7 @@ public void test_build_longMetrics() throws IOException { builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(8, resultStarTreeDocuments.size()); + assertEquals(7, resultStarTreeDocuments.size()); Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); @@ -517,8 +508,7 @@ private static Iterator getExpectedStarTreeDocumentIterator() new StarTreeDocument(new Long[] { -1L, 4L, 3L, 4L }, new Object[] { 21.0, 14.0, 2L }), new StarTreeDocument(new Long[] { -1L, 4L, -1L, 1L }, new Object[] { 35.0, 34.0, 3L }), new StarTreeDocument(new Long[] { -1L, 4L, -1L, 4L }, new Object[] { 21.0, 14.0, 2L }), - new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L }), - new StarTreeDocument(new Long[] { -1L, -1L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L }) + new StarTreeDocument(new Long[] { -1L, 4L, -1L, -1L }, new Object[] { 56.0, 48.0, 5L }) ); return expectedStarTreeDocuments.iterator(); } @@ -546,13 +536,13 @@ public void test_build() throws IOException { builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); - assertEquals(8, resultStarTreeDocuments.size()); + assertEquals(7, resultStarTreeDocuments.size()); Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); } - private static void assertStarTreeDocuments( + private void assertStarTreeDocuments( List resultStarTreeDocuments, Iterator expectedStarTreeDocumentIterator ) { @@ -571,6 +561,142 @@ private static void assertStarTreeDocuments( } } + public void test_build_starTreeDataset() throws IOException { + + fields = List.of("fieldC", "fieldB", "fieldL", "fieldI"); + + dimensionsOrder = List.of(new NumericDimension("fieldC"), new NumericDimension("fieldB"), new NumericDimension("fieldL")); + metrics = List.of(new Metric("fieldI", List.of(MetricStat.SUM))); + + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of(), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + ); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 7, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + fieldProducerMap = new HashMap<>(); + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + fieldProducerMap.put(fields.get(i), docValuesProducer); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("fieldI", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); + + int noOfStarTreeDocuments = 7; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + starTreeDocuments[0] = new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Double[] { 400.0 }); + starTreeDocuments[1] = new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Double[] { 200.0 }); + starTreeDocuments[2] = new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Double[] { 300.0 }); + starTreeDocuments[3] = new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Double[] { 100.0 }); + starTreeDocuments[4] = new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Double[] { 600.0 }); + starTreeDocuments[5] = new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Double[] { 200.0 }); + starTreeDocuments[6] = new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Double[] { 400.0 }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateStarTreeDocuments(segmentStarTreeDocuments); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + List expectedStarTreeDocuments = List.of( + new StarTreeDocument(new Long[] { 1L, 11L, 21L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { 1L, 12L, 22L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { 2L, 13L, 21L }, new Object[] { 100.0 }), + new StarTreeDocument(new Long[] { 2L, 13L, 23L }, new Object[] { 300.0 }), + new StarTreeDocument(new Long[] { 3L, 11L, 21L }, new Object[] { 600.0 }), + new StarTreeDocument(new Long[] { 3L, 12L, 21L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { 3L, 12L, 23L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { -1L, 11L, 21L }, new Object[] { 1000.0 }), + new StarTreeDocument(new Long[] { -1L, 12L, 21L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { -1L, 12L, 22L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { -1L, 12L, 23L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { -1L, 13L, 21L }, new Object[] { 100.0 }), + new StarTreeDocument(new Long[] { -1L, 13L, 23L }, new Object[] { 300.0 }), + new StarTreeDocument(new Long[] { -1L, -1L, 21L }, new Object[] { 1500.0 }), + new StarTreeDocument(new Long[] { -1L, -1L, 22L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { -1L, -1L, 23L }, new Object[] { 500.0 }), + new StarTreeDocument(new Long[] { -1L, -1L, -1L }, new Object[] { 2200.0 }), + new StarTreeDocument(new Long[] { -1L, 12L, -1L }, new Object[] { 800.0 }), + new StarTreeDocument(new Long[] { -1L, 13L, -1L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { 1L, -1L, 21L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { 1L, -1L, 22L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { 1L, -1L, -1L }, new Object[] { 600.0 }), + new StarTreeDocument(new Long[] { 2L, 13L, -1L }, new Object[] { 400.0 }), + new StarTreeDocument(new Long[] { 3L, -1L, 21L }, new Object[] { 1000.0 }), + new StarTreeDocument(new Long[] { 3L, -1L, 23L }, new Object[] { 200.0 }), + new StarTreeDocument(new Long[] { 3L, -1L, -1L }, new Object[] { 1200.0 }), + new StarTreeDocument(new Long[] { 3L, 12L, -1L }, new Object[] { 600.0 }) + ); + + Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator(); + Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); + while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + } + } + @Override public void tearDown() throws Exception { super.tearDown(); From 081f8b8c9a0bed8027096154a1de0fd99eb349b5 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 17 Jul 2024 09:45:43 +0530 Subject: [PATCH 16/16] nits Signed-off-by: Sarthak Aggarwal --- CHANGELOG.md | 1 - .../datacube/startree/builder/BaseStarTreeBuilder.java | 2 +- .../datacube/startree/builder/OnHeapStarTreeBuilderTests.java | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 452579c6d3a36..b863b9d13e789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604)) - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750)) - Add Plugin interface for loading application based configuration templates (([#14659](https://github.com/opensearch-project/OpenSearch/issues/14659))) -- Star Tree Implementation (OnHeap) ([#14512](https://github.com/opensearch-project/OpenSearch/pull/14512)) ### Dependencies - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442)) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index c6249a87e7c00..0a363bfad8fe1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -628,7 +628,7 @@ private StarTreeDocument createAggregatedDocs(TreeNode node) throws IOException // If no star child exists, aggregate all aggregated documents from non-star children if (node.children.values().size() == 1) { for (TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = createAggregatedDocs(child); + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); node.aggregatedDocId = child.aggregatedDocId; } } else { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 941fb4abfbe21..4e107e78d27be 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -695,6 +695,7 @@ public void test_build_starTreeDataset() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); } + } @Override