From 206f66fa6474add99c035c92a052f9dff1804b59 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 8 Jul 2024 08:18:02 +0530 Subject: [PATCH] star-tree file formats reader and javadoc fixes --- .../lucene/index/BaseStarTreeBuilder.java | 80 ++++++++---- .../composite/Composite90DocValuesFormat.java | 24 ++++ .../composite/Composite90DocValuesReader.java | 116 ++++++++++++++++-- .../composite/Composite90DocValuesWriter.java | 68 +++++++++- .../composite/CompositeCodecFactory.java | 10 +- .../codec/composite/CompositeIndexValues.java | 2 +- .../CompositeIndexConstants.java | 12 +- .../CompositeIndexMetadata.java | 60 +++++++-- .../datacube/startree/StarTreeDocument.java | 1 + .../aggregators/MetricAggregatorInfo.java | 2 +- .../startree/aggregators/MetricEntry.java | 5 + .../startree/aggregators/ValueAggregator.java | 1 - .../numerictype/StarTreeNumericType.java | 2 +- .../StarTreeNumericTypeConverters.java | 1 + .../aggregators/numerictype/package-info.java | 1 + .../startree/aggregators/package-info.java | 1 + .../builder/OnHeapStarTreeBuilder.java | 13 +- .../startree/builder/StarTreesBuilder.java | 26 ++-- .../startree/builder/package-info.java | 1 + .../startree/meta/StarTreeMetadata.java | 46 ++++++- .../datacube/startree/meta/TreeMetadata.java | 50 +++++++- .../datacube/startree/meta/package-info.java | 2 + .../startree/node/OffHeapStarTree.java | 18 ++- .../startree/node/OffHeapStarTreeNode.java | 4 +- .../datacube/startree/node/StarTree.java | 17 ++- .../datacube/startree/node/StarTreeNode.java | 76 ++++++++++-- .../datacube/startree/node/package-info.java | 2 + .../datacube/startree/package-info.java | 2 + .../startree/utils/StarTreeBuilderUtils.java | 9 +- .../utils/StarTreeDataSerializer.java | 42 +++++++ .../utils/StarTreeMetaSerializer.java | 65 +++++++++- .../datacube/startree/utils/package-info.java | 1 + .../index/compositeindex/package-info.java | 3 +- .../builder/BaseStarTreeBuilderTests.java | 28 ++++- .../builder/OnHeapStarTreeBuilderTests.java | 33 ++++- 35 files changed, 695 insertions(+), 129 deletions(-) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 642116afbde96..2d0e693bb7c60 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.store.IndexOutput; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; @@ -67,19 +68,31 @@ public abstract class BaseStarTreeBuilder implements StarTreeBuilder { private final StarTreeField starTreeField; private final MapperService mapperService; - private final SegmentWriteState state; + private final SegmentWriteState writeState; + + private final IndexOutput metaOut; + private final IndexOutput dataOut; /** * Builds star tree based on star tree field configuration consisting of dimensions, metrics and star tree index specific configuration. * - * @param starTreeField holds the configuration for the star tree - * @param state stores the segment write state - * @param mapperService helps to find the original type of the field + * @param starTreeField holds the configuration for the star tree + * @param writeState stores the segment write writeState + * @param mapperService helps to find the original type of the field */ - protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) throws IOException { + protected BaseStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState writeState, + MapperService mapperService + ) throws IOException { logger.debug("Building star tree : {}", starTreeField); + this.metaOut = metaOut; + this.dataOut = dataOut; + this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); @@ -87,9 +100,9 @@ protected BaseStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState sta this.numDimensions = dimensionsSplitOrder.size(); this.skipStarNodeCreationForDimensions = new HashSet<>(); - this.totalSegmentDocs = state.segmentInfo.maxDoc(); + this.totalSegmentDocs = writeState.segmentInfo.maxDoc(); this.mapperService = mapperService; - this.state = state; + this.writeState = writeState; Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); @@ -142,19 +155,20 @@ public List generateMetricAggregatorInfos(MapperService ma */ public List getMetricReaders(SegmentWriteState state, Map fieldProducerMap) throws IOException { + List metricReaders = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { - SequentialDocValuesIterator metricReader = null; - + SequentialDocValuesIterator metricReader; FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); - // TODO - // if (metricType != MetricStat.COUNT) { - // Need not initialize the metric reader for COUNT metric type - metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) - ); - // } + if (metricType != MetricStat.COUNT) { + // Need not initialize the metric reader with relevant doc id set iterator for COUNT metric type + metricReader = new SequentialDocValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ); + } else { + metricReader = new SequentialDocValuesIterator(); + } metricReaders.add(metricReader); } @@ -166,19 +180,18 @@ public List getMetricReaders(SegmentWriteState stat * Builds the star tree from the original segment documents * * @param fieldProducerMap contain s the docValues producer to get docValues associated with each field - * * @throws IOException when we are unable to build star-tree */ public void build(Map fieldProducerMap) throws IOException { long startTime = System.currentTimeMillis(); logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - List metricReaders = getMetricReaders(state, fieldProducerMap); + List metricReaders = getMetricReaders(writeState, fieldProducerMap); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; for (int i = 0; i < numDimensions; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - FieldInfo dimensionFieldInfo = state.fieldInfos.fieldInfo(dimension); + FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); dimensionReaders[i] = new SequentialDocValuesIterator( fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) ); @@ -209,8 +222,8 @@ public void build(Iterator starTreeDocumentIterator) throws IO logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); if (numStarTreeDocs == 0) { - // TODO: Uncomment when segment codec is ready - // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + // serialize the star tree data + serializeStarTree(numSegmentStarTreeDocument); return; } @@ -228,10 +241,29 @@ public void build(Iterator starTreeDocumentIterator) throws IO // TODO: When StarTree Codec is ready // Create doc values indices in disk - // Serialize and save in disk + + serializeStarTree(numSegmentStarTreeDocument); + // Write star tree metadata for off heap implementation } + private void serializeStarTree(int numSegmentStarTreeDocument) throws IOException { + // serialize the star tree data + long dataFilePointer = dataOut.getFilePointer(); + long totalStarTreeDataLength = StarTreeBuilderUtils.serializeStarTree(dataOut, rootNode, numStarTreeNodes); + + // serialize the star tree meta + StarTreeBuilderUtils.serializeStarTreeMetadata( + metaOut, + starTreeField, + writeState, + metricAggregatorInfos, + numSegmentStarTreeDocument, + dataFilePointer, + totalStarTreeDataLength + ); + } + /** * Adds a document to the star-tree. * @@ -269,9 +301,9 @@ public void build(Iterator starTreeDocumentIterator) throws IO * Sorts and aggregates all the documents in the segment as per the configuration, and returns a star-tree document iterator for all the * aggregated star-tree documents. * - * @param numDocs number of documents in the given segment + * @param numDocs number of documents in the given segment * @param dimensionReaders List of docValues readers to read dimensions from the segment - * @param metricReaders List of docValues readers to read metrics from the segment + * @param metricReaders List of docValues readers to read metrics from the segment * @return Iterator for the aggregated star-tree document */ public abstract Iterator sortAndAggregateSegmentDocuments( diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java index 59ee4c4aa64da..3643eeeff27eb 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesFormat.java @@ -37,6 +37,30 @@ public class Composite90DocValuesFormat extends DocValuesFormat { private final DocValuesFormat delegate; private final MapperService mapperService; + /** Data codec name for Composite Doc Values Format */ + public static final String DATA_CODEC_NAME = "Composite90FormatData"; + + /** Meta codec name for Composite Doc Values Format */ + public static final String META_CODEC_NAME = "Composite90FormatMeta"; + + /** Filename extension for the composite index data */ + public static final String DATA_EXTENSION = "sttd"; + + /** Filename extension for the composite index meta */ + public static final String META_EXTENSION = "sttm"; + + /** Filename extension for the composite index data doc values */ + public static final String DATA_DOC_VALUES_EXTENSION = "sttddvm"; + + /** Filename extension for the composite index meta doc values */ + public static final String META_DOC_VALUES_EXTENSION = "sttmdvm"; + + /** Initial version for the Composite90DocValuesFormat */ + public static final int VERSION_START = 0; + + /** Current version for the Composite90DocValuesFormat */ + public static final int VERSION_CURRENT = VERSION_START; + // needed for SPI public Composite90DocValuesFormat() { this(new Lucene90DocValuesFormat(), null); diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java index 8a74df2f52c10..d75350c2a8173 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesReader.java @@ -8,22 +8,32 @@ package org.opensearch.index.codec.composite; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.mapper.CompositeMappedFieldType; -import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.compositeindex.CompositeIndexMetadata; +import org.opensearch.index.compositeindex.datacube.startree.node.OffHeapStarTree; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTree; import java.io.IOException; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Set; +import java.util.Map; /** * Reader for star tree index and star tree doc values from the segments @@ -32,11 +42,95 @@ */ @ExperimentalApi public class Composite90DocValuesReader extends DocValuesProducer implements CompositeIndexReader { - private DocValuesProducer delegate; + private static final Logger logger = LogManager.getLogger(CompositeIndexMetadata.class); - public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState state) throws IOException { + private final DocValuesProducer delegate; + private final IndexInput dataIn; + private final ChecksumIndexInput metaIn; + private final Map starTreeMap = new LinkedHashMap<>(); + private final Map starTreeMetaMap = new LinkedHashMap<>(); + private final List compositeFieldInfos = new ArrayList<>(); + + public Composite90DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException { this.delegate = producer; - // TODO : read star tree files + + String metaFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Composite90DocValuesFormat.META_EXTENSION + ); + + String dataFileName = IndexFileNames.segmentFileName( + readState.segmentInfo.name, + readState.segmentSuffix, + Composite90DocValuesFormat.DATA_EXTENSION + ); + + boolean success = false; + try { + + dataIn = readState.directory.openInput(dataFileName, readState.context); + CodecUtil.checkIndexHeader( + dataIn, + Composite90DocValuesFormat.DATA_CODEC_NAME, + Composite90DocValuesFormat.VERSION_START, + Composite90DocValuesFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + CodecUtil.retrieveChecksum(dataIn); + + metaIn = readState.directory.openChecksumInput(metaFileName, readState.context); + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader( + metaIn, + Composite90DocValuesFormat.META_CODEC_NAME, + Composite90DocValuesFormat.VERSION_START, + Composite90DocValuesFormat.VERSION_CURRENT, + readState.segmentInfo.getId(), + readState.segmentSuffix + ); + + while (true) { + long magicMarker = metaIn.readLong(); + + if (magicMarker == -1) { + logger.info("EOF reached for composite index metadata"); + return; + } else if (magicMarker < 0) { + throw new CorruptIndexException("Unknown token encountered: " + magicMarker, metaIn); + } + CompositeIndexMetadata compositeIndexMetadata = new CompositeIndexMetadata(metaIn, magicMarker); + compositeFieldInfos.add( + new CompositeIndexFieldInfo( + compositeIndexMetadata.getCompositeFieldName(), + compositeIndexMetadata.getCompositeFieldType() + ) + ); + switch (compositeIndexMetadata.getCompositeFieldType()) { + case STAR_TREE: + StarTree starTree = new OffHeapStarTree(dataIn, compositeIndexMetadata.getStarTreeMetadata()); + starTreeMap.put(compositeIndexMetadata.getCompositeFieldName(), starTree); + starTreeMetaMap.put(compositeIndexMetadata.getCompositeFieldName(), compositeIndexMetadata); + break; + default: + throw new CorruptIndexException("Invalid composite field type found in the file", dataIn); + } + } + } catch (Throwable t) { + priorE = t; + } finally { + CodecUtil.checkFooter(metaIn, priorE); + } + CodecUtil.retrieveChecksum(dataIn); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } @Override @@ -67,25 +161,27 @@ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { @Override public void checkIntegrity() throws IOException { delegate.checkIntegrity(); - // Todo : check integrity of composite index related [star tree] files + CodecUtil.checksumEntireFile(metaIn); + CodecUtil.checksumEntireFile(dataIn); } @Override public void close() throws IOException { delegate.close(); - // Todo: close composite index related files [star tree] files + starTreeMap.clear(); + starTreeMetaMap.clear(); } @Override public List getCompositeIndexFields() { - // todo : read from file formats and get the field names. - return new ArrayList<>(); + return compositeFieldInfos; } @Override public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo compositeIndexFieldInfo) throws IOException { // TODO : read compositeIndexValues [starTreeValues] from star tree files + throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java index 00d6f4fbd03ea..97dec4bd55c88 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/Composite90DocValuesWriter.java @@ -10,11 +10,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -45,12 +49,52 @@ public class Composite90DocValuesWriter extends DocValuesConsumer { private final Set compositeMappedFieldTypes; private final Set compositeFieldSet; + public IndexOutput dataOut; + public IndexOutput metaOut; + private final Map fieldProducerMap = new HashMap<>(); private static final Logger logger = LogManager.getLogger(Composite90DocValuesWriter.class); public Composite90DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) throws IOException { + boolean success = false; + try { + String dataFileName = IndexFileNames.segmentFileName( + segmentWriteState.segmentInfo.name, + segmentWriteState.segmentSuffix, + Composite90DocValuesFormat.DATA_EXTENSION + ); + dataOut = segmentWriteState.directory.createOutput(dataFileName, segmentWriteState.context); + CodecUtil.writeIndexHeader( + dataOut, + Composite90DocValuesFormat.DATA_CODEC_NAME, + Composite90DocValuesFormat.VERSION_CURRENT, + segmentWriteState.segmentInfo.getId(), + segmentWriteState.segmentSuffix + ); + + String metaFileName = IndexFileNames.segmentFileName( + segmentWriteState.segmentInfo.name, + segmentWriteState.segmentSuffix, + Composite90DocValuesFormat.META_EXTENSION + ); + metaOut = segmentWriteState.directory.createOutput(metaFileName, segmentWriteState.context); + CodecUtil.writeIndexHeader( + metaOut, + Composite90DocValuesFormat.META_CODEC_NAME, + Composite90DocValuesFormat.VERSION_CURRENT, + segmentWriteState.segmentInfo.getId(), + segmentWriteState.segmentSuffix + ); + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + this.delegate = delegate; this.state = segmentWriteState; this.mapperService = mapperService; @@ -93,6 +137,24 @@ public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) @Override public void close() throws IOException { delegate.close(); + boolean success = false; + try { + if (metaOut != null) { + metaOut.writeLong(-1); // write EOF marker + CodecUtil.writeFooter(metaOut); // write checksum + } + if (dataOut != null) { + CodecUtil.writeFooter(dataOut); // write checksum + } + success = true; + } finally { + if (success) { + IOUtils.close(dataOut, metaOut); + } else { + IOUtils.closeWhileHandlingException(dataOut, metaOut); + } + metaOut = dataOut = null; + } } private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, FieldInfo field) throws IOException { @@ -109,7 +171,7 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer, } } StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); - starTreesBuilder.build(fieldProducerMap); + starTreesBuilder.build(metaOut, dataOut, fieldProducerMap); } } @@ -123,6 +185,7 @@ public void merge(MergeState mergeState) throws IOException { /** * Merges composite fields from multiple segments + * * @param mergeState merge state */ private void mergeCompositeFields(MergeState mergeState) throws IOException { @@ -131,6 +194,7 @@ private void mergeCompositeFields(MergeState mergeState) throws IOException { /** * Merges star tree data fields from multiple segments + * * @param mergeState merge state */ private void mergeStarTreeFields(MergeState mergeState) throws IOException { @@ -169,6 +233,6 @@ private void mergeStarTreeFields(MergeState mergeState) throws IOException { } } final StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService); - starTreesBuilder.buildDuringMerge(starTreeFieldMap, starTreeSubsPerField); + starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeFieldMap, starTreeSubsPerField); } } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java index 0297e1e0db36b..3acedc6a27d7f 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java @@ -8,20 +8,20 @@ package org.opensearch.index.codec.composite; -import java.util.HashMap; -import java.util.Map; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.mapper.MapperService; +import java.util.HashMap; +import java.util.Map; + import static org.opensearch.index.codec.CodecService.BEST_COMPRESSION_CODEC; import static org.opensearch.index.codec.CodecService.DEFAULT_CODEC; import static org.opensearch.index.codec.CodecService.LZ4; import static org.opensearch.index.codec.CodecService.ZLIB; - /** * Factory class to return the latest composite codec for all the modes * @@ -34,8 +34,8 @@ public CompositeCodecFactory() {} public Map getCompositeIndexCodecs(MapperService mapperService, Logger logger) { Map codecs = new HashMap<>(); codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); + codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger)); return codecs; } diff --git a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java index c28d6afc143ad..f8848aceab343 100644 --- a/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java +++ b/server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexValues.java @@ -17,5 +17,5 @@ */ @ExperimentalApi public interface CompositeIndexValues { - CompositeIndexValues getValues(); + CompositeIndexValues getValues(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java index 8e17b7a634bd1..defb4a26de260 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java @@ -8,9 +8,19 @@ package org.opensearch.index.compositeindex; +/** + * This class contains constants used in the Composite Index implementation. + */ public class CompositeIndexConstants { - public static final long MAGIC_MARKER = 0xC0950513F1E1DL; // OpenSearch Tree + /** + * The magic marker value used for sanity checks in the Composite Index implementation. + */ + public static final long MAGIC_MARKER = 0xC0950513F1E1DL; // Composite Field + + /** + * The version of the Composite Index implementation. + */ public static final int VERSION = 1; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java index ebece2912672d..a3e3bbc687566 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java @@ -16,19 +16,32 @@ import org.opensearch.index.mapper.CompositeMappedFieldType; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import static org.opensearch.index.compositeindex.CompositeIndexConstants.MAGIC_MARKER; import static org.opensearch.index.compositeindex.CompositeIndexConstants.VERSION; +/** + * This class represents the metadata of a Composite Index, which includes information about + * the composite field name, type, and the specific metadata for the type of composite field + * (e.g., StarTree metadata). + * + * @opensearch.experimental + */ public class CompositeIndexMetadata { private static final Logger logger = LogManager.getLogger(CompositeIndexMetadata.class); - private final Map starTreeMetadataMap = new HashMap<>(); + private final String compositeFieldName; + private final CompositeMappedFieldType.CompositeFieldType compositeFieldType; + private final StarTreeMetadata starTreeMetadata; - public CompositeIndexMetadata(IndexInput meta) throws IOException { - long magicMarker = meta.readLong(); + /** + * Constructs a CompositeIndexMetadata object from the provided IndexInput and magic marker. + * + * @param meta the IndexInput containing the metadata + * @param magicMarker the magic marker value + * @throws IOException if an I/O error occurs while reading the metadata + */ + public CompositeIndexMetadata(IndexInput meta, long magicMarker) throws IOException { if (MAGIC_MARKER != magicMarker) { logger.error("Invalid composite field magic marker"); throw new IOException("Invalid composite field magic marker"); @@ -39,15 +52,13 @@ public CompositeIndexMetadata(IndexInput meta) throws IOException { throw new IOException("Invalid composite field version"); } - String compositeFieldName = meta.readString(); - String compositeFieldType = meta.readString(); + compositeFieldName = meta.readString(); + compositeFieldType = CompositeMappedFieldType.CompositeFieldType.fromName(meta.readString()); - CompositeMappedFieldType.CompositeFieldType fieldType = CompositeMappedFieldType.CompositeFieldType.fromName(compositeFieldType); - - switch (fieldType) { + switch (compositeFieldType) { // support for type of composite fields can be added in the future. case STAR_TREE: - starTreeMetadataMap.put(compositeFieldName, new StarTreeMetadata(meta, compositeFieldName, compositeFieldType)); + starTreeMetadata = new StarTreeMetadata(meta, compositeFieldName, compositeFieldType.getName()); break; default: throw new CorruptIndexException("Invalid composite field type present in the file", meta); @@ -55,7 +66,30 @@ public CompositeIndexMetadata(IndexInput meta) throws IOException { } - public Map getStarTreeMetadataMap() { - return starTreeMetadataMap; + /** + * Returns the star-tree metadata. + * + * @return the StarTreeMetadata + */ + public StarTreeMetadata getStarTreeMetadata() { + return starTreeMetadata; + } + + /** + * Returns the name of the composite field. + * + * @return the composite field name + */ + public String getCompositeFieldName() { + return compositeFieldName; + } + + /** + * Returns the type of the composite field. + * + * @return the composite field type + */ + public CompositeMappedFieldType.CompositeFieldType getCompositeFieldType() { + return compositeFieldType; } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java index 0fc66b27e65a9..0ce2b3a5cdac5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java @@ -14,6 +14,7 @@ /** * Star tree document + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index 69b7f35b2d314..81dd0a99bc592 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -17,6 +16,7 @@ /** * Builds aggregation function and doc values field pair to support various aggregations + * * @opensearch.experimental */ public class MetricAggregatorInfo implements Comparable { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricEntry.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricEntry.java index b4ea0bc3fd218..683153ecde689 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricEntry.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricEntry.java @@ -10,6 +10,11 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; +/** + * Holds the pair of metric name and it's associated stat + * + * @opensearch.experimental + */ public class MetricEntry { private final String metricName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index 0a8285678a36c..3dd1f85845c17 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index b836e73d96f25..57fe573a6a93c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -8,13 +8,13 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.fielddata.IndexNumericFieldData; import java.util.function.Function; /** * Enum to map Star Tree Numeric Types to Lucene's Numeric Type + * * @opensearch.experimental */ public enum StarTreeNumericType { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index ff76256e4ff91..eb7647c4f9851 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -15,6 +15,7 @@ /** * Numeric converters used during aggregations of metric values + * * @opensearch.experimental */ @ExperimentalApi diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java index b9a9bb8f1427d..18163056ec3b5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java @@ -8,6 +8,7 @@ /** * Numeric Types for Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java index 27565ffded2cf..bddd6a46fbbe8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java @@ -8,6 +8,7 @@ /** * Aggregators for Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.aggregators; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index bee142bd95994..00cce31dc0771 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; @@ -28,6 +29,7 @@ /** * On heap based single tree builder + * * @opensearch.experimental */ @ExperimentalApi @@ -43,9 +45,14 @@ public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { * @param mapperService helps with the numeric type of field * @throws IOException throws an exception when we are unable to construct a star-tree using on-heap approach */ - public OnHeapStarTreeBuilder(StarTreeField starTreeField, SegmentWriteState segmentWriteState, MapperService mapperService) - throws IOException { - super(starTreeField, segmentWriteState, mapperService); + public OnHeapStarTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) throws IOException { + super(metaOut, dataOut, starTreeField, segmentWriteState, mapperService); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index dc24fb8d8f745..6cb4ca81e07c8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; @@ -63,7 +64,7 @@ public StarTreesBuilder(SegmentWriteState segmentWriteState, MapperService mappe /** * Builds the star-trees. */ - public void build(Map fieldProducerMap) throws IOException { + public void build(IndexOutput metaOut, IndexOutput dataOut, Map fieldProducerMap) throws IOException { if (starTreeFields.isEmpty()) { logger.debug("no star-tree fields found, returning from star-tree builder"); return; @@ -76,7 +77,7 @@ public void build(Map fieldProducerMap) throws IOExce // Build all star-trees for (int i = 0; i < numStarTrees; i++) { StarTreeField starTreeField = starTreeFields.get(i); - try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(starTreeField, state, mapperService)) { + try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService)) { starTreeBuilder.build(fieldProducerMap); } } @@ -90,18 +91,22 @@ public void close() throws IOException { /** * Merges star tree fields from multiple segments - * @param starTreeFieldMap StarTreeField configuration per field - * @param starTreeValuesSubsPerField starTreeValuesSubs per field * + * @param metaOut + * @param dataOut + * @param starTreeFieldMap StarTreeField configuration per field + * @param starTreeValuesSubsPerField starTreeValuesSubs per field */ public void buildDuringMerge( + IndexOutput metaOut, + IndexOutput dataOut, final Map starTreeFieldMap, final Map> starTreeValuesSubsPerField ) throws IOException { for (Map.Entry> entry : starTreeValuesSubsPerField.entrySet()) { List starTreeValuesList = entry.getValue(); StarTreeField starTreeField = starTreeFieldMap.get(entry.getKey()); - StarTreeBuilder builder = getSingleTreeBuilder(starTreeField, state, mapperService); + StarTreeBuilder builder = getSingleTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService); builder.build(starTreeValuesList); } } @@ -109,11 +114,16 @@ public void buildDuringMerge( /** * Get star-tree builder based on build mode. */ - private static StarTreeBuilder getSingleTreeBuilder(StarTreeField starTreeField, SegmentWriteState state, MapperService mapperService) - throws IOException { + private static StarTreeBuilder getSingleTreeBuilder( + IndexOutput metaOut, + IndexOutput dataOut, + StarTreeField starTreeField, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { switch (starTreeField.getStarTreeConfig().getBuildMode()) { case ON_HEAP: - return new OnHeapStarTreeBuilder(starTreeField, state, mapperService); + return new OnHeapStarTreeBuilder(metaOut, dataOut, starTreeField, state, mapperService); default: throw new IllegalArgumentException( String.format( diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java index 80eed545ef8a5..9c97b076371a3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java @@ -8,6 +8,7 @@ /** * Builders for Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/StarTreeMetadata.java index 0735a5b52c75c..c2585dc6e79eb 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/StarTreeMetadata.java @@ -13,14 +13,15 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricEntry; -import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import java.io.IOException; import java.util.ArrayList; import java.util.List; /** - * Off heap implementation of {@link StarTreeNode} + * Holds the associated metadata for the building of star-tree + * + * @opensearch.experimental */ public class StarTreeMetadata implements TreeMetadata { private static final Logger logger = LogManager.getLogger(TreeMetadata.class); @@ -39,7 +40,7 @@ public StarTreeMetadata(IndexInput meta, String compositeFieldName, String compo this.starTreeFieldName = compositeFieldName; this.starTreeFieldType = compositeFieldType; this.dimensionOrdinals = readStarTreeDimensions(); - this.metricEntries = readMetricPairs(); + this.metricEntries = readMetricEntries(); this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); this.dataStartFilePointer = readDataStartFilePointer(); this.dataLength = readDataLength(); @@ -72,7 +73,7 @@ public int readMetricsCount() throws IOException { } @Override - public List readMetricPairs() throws IOException { + public List readMetricEntries() throws IOException { int metricCount = readMetricsCount(); List metricEntries = new ArrayList<>(); @@ -100,30 +101,65 @@ public long readDataLength() throws IOException { return meta.readLong(); } + /** + * Returns the name of the star-tree field. + * + * @return star-tree field name + */ public String getStarTreeFieldName() { return starTreeFieldName; } + /** + * Returns the type of the star tree field. + * + * @return star-tree field type + */ public String getStarTreeFieldType() { return starTreeFieldType; } + /** + * Returns the list of dimension ordinals. + * + * @return star-tree dimension ordinals + */ public List getDimensionOrdinals() { return dimensionOrdinals; } - public List getMetricPairs() { + /** + * Returns the list of metric entries. + * + * @return star-tree metric entries + */ + public List getMetricEntries() { return metricEntries; } + /** + * Returns the aggregated document count for the star-tree. + * + * @return the aggregated document count for the star-tree. + */ public Integer getSegmentAggregatedDocCount() { return segmentAggregatedDocCount; } + /** + * Returns the file pointer to the start of the star-tree data. + * + * @return start file pointer for star-tree data + */ public long getDataStartFilePointer() { return dataStartFilePointer; } + /** + * Returns the length of star-tree data + * + * @return star-tree length + */ public long getDataLength() { return dataLength; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/TreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/TreeMetadata.java index 61b9a3108e282..f9c8db85ee17c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/TreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/TreeMetadata.java @@ -13,20 +13,66 @@ import java.io.IOException; import java.util.List; +/** + * An interface for metadata of the star-tree + * + * @opensearch.experimental + */ public interface TreeMetadata { + /** + * Reads the count of dimensions in the star-tree. + * + * @return the count of dimensions + * @throws IOException if an I/O error occurs while reading the dimensions count + */ int readDimensionsCount() throws IOException; + /** + * Reads the list of dimension ordinals in the star-tree. + * + * @return the list of dimension ordinals + * @throws IOException if an I/O error occurs while reading the dimension ordinals + */ List readStarTreeDimensions() throws IOException; + /** + * Reads the count of metrics in the star-tree. + * + * @return the count of metrics + * @throws IOException if an I/O error occurs while reading the metrics count + */ int readMetricsCount() throws IOException; - List readMetricPairs() throws IOException; + /** + * Reads the list of metric entries in the star-tree. + * + * @return the list of metric entries + * @throws IOException if an I/O error occurs while reading the metric entries + */ + List readMetricEntries() throws IOException; + /** + * Reads the aggregated document count for the segment in the star-tree. + * + * @return the aggregated document count for the segment + * @throws IOException if an I/O error occurs while reading the aggregated document count + */ int readSegmentAggregatedDocCount() throws IOException; + /** + * Reads the file pointer to the start of the star-tree data. + * + * @return the file pointer to the start of the star-tree data + * @throws IOException if an I/O error occurs while reading the star-tree data start file pointer + */ long readDataStartFilePointer() throws IOException; + /** + * Reads the length of the data of the star-tree. + * + * @return the length of the data + * @throws IOException if an I/O error occurs while reading the data length + */ long readDataLength() throws IOException; - } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/package-info.java index d7728dd544cf3..568cacea4b59a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/package-info.java @@ -8,5 +8,7 @@ /** * Meta package for star tree + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.meta; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java index 610897c1771ab..1f16bd924dac6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java @@ -14,19 +14,18 @@ import org.opensearch.index.compositeindex.datacube.startree.meta.StarTreeMetadata; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import static org.opensearch.index.compositeindex.CompositeIndexConstants.MAGIC_MARKER; import static org.opensearch.index.compositeindex.CompositeIndexConstants.VERSION; /** - * Off heap implementation of star tree. + * Off heap implementation of the star-tree. + * + * @opensearch.experimental */ public class OffHeapStarTree implements StarTree { private static final Logger logger = LogManager.getLogger(OffHeapStarTree.class); private final OffHeapStarTreeNode root; - private final List dimensionNames = new ArrayList<>(); private final Integer numNodes; public OffHeapStarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throws IOException { @@ -42,7 +41,6 @@ public OffHeapStarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throw } numNodes = data.readInt(); // num nodes - // should we get start and end file pointer from meta file? RandomAccessInput in = data.randomAccessSlice( starTreeMetadata.getDataStartFilePointer(), starTreeMetadata.getDataStartFilePointer() + starTreeMetadata.getDataLength() @@ -55,11 +53,11 @@ public StarTreeNode getRoot() { return root; } - @Override - public List getDimensionNames() { - return dimensionNames; - } - + /** + * Returns the number of nodes in star-tree + * + * @return number of nodes in te star-tree + */ public Integer getNumNodes() { return numNodes; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java index b1e4f1918f20f..21da0a8e71d55 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTreeNode.java @@ -12,10 +12,10 @@ import java.io.IOException; import java.util.Iterator; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils.ALL; - /** * Off heap implementation of {@link StarTreeNode} + * + * @opensearch.experimental */ public class OffHeapStarTreeNode implements StarTreeNode { public static final int NUM_INT_SERIALIZABLE_FIELDS = 7; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java index 0fe75ce8264cd..e21dc225a4c8f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTree.java @@ -7,18 +7,17 @@ */ package org.opensearch.index.compositeindex.datacube.startree.node; -import java.util.List; - -/** Interface for star tree */ +/** + * Interface for star-tree. + * + * @opensearch.experimental + */ public interface StarTree { - /** Get the root node of the star tree. */ - StarTreeNode getRoot(); - /** - * Get a list of all dimension names. The node dimension id is the index of the dimension name in - * this list. + * Fetches the root node of the star-tree. + * @return the root of the star-tree */ - List getDimensionNames(); + StarTreeNode getRoot(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index befca66ab8c83..59522ffa4be89 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -22,39 +22,91 @@ public interface StarTreeNode { long ALL = -1l; - /** Get the index of the dimension. */ + /** + * Returns the dimension ID of the current star-tree node. + * + * @return the dimension ID + * @throws IOException if an I/O error occurs while reading the dimension ID + */ int getDimensionId() throws IOException; - /** Get the value (dictionary id) of the dimension. */ + /** + * Returns the dimension value of the current star-tree node. + * + * @return the dimension value + * @throws IOException if an I/O error occurs while reading the dimension value + */ long getDimensionValue() throws IOException; - /** Get the child dimension id. */ + /** + * Returns the dimension ID of the child star-tree node. + * + * @return the child dimension ID + * @throws IOException if an I/O error occurs while reading the child dimension ID + */ int getChildDimensionId() throws IOException; - /** Get the index of the start document. */ + /** + * Returns the start document ID of the current star-tree node. + * + * @return the start document ID + * @throws IOException if an I/O error occurs while reading the start document ID + */ int getStartDocId() throws IOException; - /** Get the index of the end document (exclusive). */ + /** + * Returns the end document ID of the current star-tree node. + * + * @return the end document ID + * @throws IOException if an I/O error occurs while reading the end document ID + */ int getEndDocId() throws IOException; - /** Get the index of the aggregated document. */ + /** + * Returns the aggregated document ID of the current star-tree node. + * + * @return the aggregated document ID + * @throws IOException if an I/O error occurs while reading the aggregated document ID + */ int getAggregatedDocId() throws IOException; - /** Get the number of children nodes. */ + /** + * Returns the number of children of the current star-tree node. + * + * @return the number of children + * @throws IOException if an I/O error occurs while reading the number of children + */ int getNumChildren() throws IOException; - /** Return true if the node is a leaf node, false otherwise. */ + /** + * Checks if the current node is a leaf star-tree node. + * + * @return true if the node is a leaf node, false otherwise + */ boolean isLeaf(); - /** Return true if the node is a star node, false otherwise. */ + /** + * Checks if the current node is a star node. + * + * @return true if the node is a star node, false otherwise + * @throws IOException if an I/O error occurs while reading the star node status + */ boolean isStarNode() throws IOException; /** - * Get the child node corresponding to the given dimension value (dictionary id), or null if such - * child does not exist. + * Returns the child star-tree node for the given dimension value. + * + * @param dimensionValue the dimension value + * @return the child node for the given dimension value or null if child is not present + * @throws IOException if an I/O error occurs while retrieving the child node */ StarTreeNode getChildForDimensionValue(long dimensionValue) throws IOException; - /** Get the iterator over all children nodes. */ + /** + * Returns an iterator over the children of the current star-tree node. + * + * @return an iterator over the children + * @throws IOException if an I/O error occurs while retrieving the children iterator + */ Iterator getChildrenIterator() throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java index 516d5b5a012ab..19d12bc6318d7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java @@ -8,5 +8,7 @@ /** * Holds classes associated with star tree node + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java index 4f4e670478e2f..6d6cb420f4a9e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java @@ -7,5 +7,7 @@ */ /** * Core classes for handling star tree index. + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index d2a7c01a0bf21..4e12f8e60d449 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -20,6 +20,7 @@ /** * Util class for building star tree + * * @opensearch.experimental */ public class StarTreeBuilderUtils { @@ -81,16 +82,16 @@ public static long serializeStarTree(IndexOutput dataOut, TreeNode rootNode, int } public static void serializeStarTreeMetadata( - IndexOutput dataOut, + IndexOutput metaOut, StarTreeField starTreeField, SegmentWriteState writeState, List metricAggregatorInfos, Integer segmentAggregatedCount, - Integer dataFilePointer, - Integer dataFileLength + long dataFilePointer, + long dataFileLength ) throws IOException { StarTreeMetaSerializer.serializeStarTreeMetadata( - dataOut, + metaOut, CompositeMappedFieldType.CompositeFieldType.STAR_TREE, starTreeField, writeState, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDataSerializer.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDataSerializer.java index f3dffe45a9680..b4f8d0955b351 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDataSerializer.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeDataSerializer.java @@ -24,10 +24,24 @@ import static org.opensearch.index.compositeindex.datacube.startree.node.OffHeapStarTreeNode.SERIALIZABLE_DATA_SIZE_IN_BYTES; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils.ALL; +/** + * Utility class for serializing a star-tree data structure. + * + * @opensearch.experimental + */ public class StarTreeDataSerializer { private static final Logger logger = LogManager.getLogger(StarTreeDataSerializer.class); + /** + * Serializes the star-tree data structure. + * + * @param indexOutput the IndexOutput to write the star-tree data + * @param rootNode the root node of the star-tree + * @param numNodes the total number of nodes in the star-tree + * @return the total size in bytes of the serialized star-tree data + * @throws IOException if an I/O error occurs while writing the star-tree data + */ public static long serializeStarTree(IndexOutput indexOutput, StarTreeBuilderUtils.TreeNode rootNode, int numNodes) throws IOException { int headerSizeInBytes = computeStarTreeDataHeaderByteSize(); long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_DATA_SIZE_IN_BYTES; @@ -39,6 +53,11 @@ public static long serializeStarTree(IndexOutput indexOutput, StarTreeBuilderUti return totalSizeInBytes; } + /** + * Computes the byte size of the star-tree data header. + * + * @return the byte size of the star-tree data header + */ private static int computeStarTreeDataHeaderByteSize() { // Magic marker (8), version (4) int headerSizeInBytes = 12; @@ -48,12 +67,26 @@ private static int computeStarTreeDataHeaderByteSize() { return headerSizeInBytes; } + /** + * Writes the star-tree data header. + * + * @param output the IndexOutput to write the header + * @param numNodes the total number of nodes in the star-tree + * @throws IOException if an I/O error occurs while writing the header + */ private static void writeStarTreeHeader(IndexOutput output, int numNodes) throws IOException { output.writeLong(MAGIC_MARKER); output.writeInt(VERSION); output.writeInt(numNodes); } + /** + * Writes the star-tree nodes in a breadth-first order. + * + * @param output the IndexOutput to write the nodes + * @param rootNode the root node of the star-tree + * @throws IOException if an I/O error occurs while writing the nodes + */ private static void writeStarTreeNodes(IndexOutput output, StarTreeBuilderUtils.TreeNode rootNode) throws IOException { Queue queue = new LinkedList<>(); queue.add(rootNode); @@ -81,6 +114,15 @@ private static void writeStarTreeNodes(IndexOutput output, StarTreeBuilderUtils. } } + /** + * Writes a single star-tree node + * + * @param output the IndexOutput to write the node + * @param node the star tree node to write + * @param firstChildId the ID of the first child node + * @param lastChildId the ID of the last child node + * @throws IOException if an I/O error occurs while writing the node + */ private static void writeStarTreeNode(IndexOutput output, StarTreeBuilderUtils.TreeNode node, int firstChildId, int lastChildId) throws IOException { output.writeInt(node.dimensionId); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java index 31e1e404be89e..4241642cf0ba5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeMetaSerializer.java @@ -24,10 +24,30 @@ import static org.opensearch.index.compositeindex.CompositeIndexConstants.MAGIC_MARKER; import static org.opensearch.index.compositeindex.CompositeIndexConstants.VERSION; +/** + * The utility class for serializing the metadata of a star-tree data structure. + * The metadata includes information about the dimensions, metrics, and other relevant details + * related to the star tree. + * + * @opensearch.experimental + */ public class StarTreeMetaSerializer { private static final Logger logger = LogManager.getLogger(StarTreeMetaSerializer.class); + /** + * Serializes the star-tree metadata. + * + * @param metaOut the IndexOutput to write the metadata + * @param compositeFieldType the composite field type of the star-tree field + * @param starTreeField the star-tree field + * @param writeState the segment write state + * @param metricAggregatorInfos the list of metric aggregator information + * @param segmentAggregatedCount the aggregated document count for the segment + * @param dataFilePointer the file pointer to the start of the star tree data + * @param dataFileLength the length of the star tree data file + * @throws IOException if an I/O error occurs while serializing the metadata + */ public static void serializeStarTreeMetadata( IndexOutput metaOut, CompositeMappedFieldType.CompositeFieldType compositeFieldType, @@ -44,7 +64,7 @@ public static void serializeStarTreeMetadata( totalSizeInBytes += Integer.BYTES; // number of dimensions totalSizeInBytes += (long) starTreeField.getDimensionsOrder().size() * Integer.BYTES; // dimension ids totalSizeInBytes += Integer.BYTES; // metric count - totalSizeInBytes += computeMetricPairSizeInBytes(metricAggregatorInfos); // metric - metric stat pair + totalSizeInBytes += computeMetricEntriesSizeInBytes(metricAggregatorInfos); // metric - metric stat pair totalSizeInBytes += Integer.BYTES; // segment aggregated document count totalSizeInBytes += Long.BYTES; // data start file pointer totalSizeInBytes += Long.BYTES; // data length @@ -55,18 +75,31 @@ public static void serializeStarTreeMetadata( writeMeta(metaOut, writeState, metricAggregatorInfos, starTreeField, segmentAggregatedCount, dataFilePointer, dataFileLength); } - private static long computeMetricPairSizeInBytes(List metricAggregatorInfos) { + /** + * Computes the byte size required to store the star-tree metric entry. + * + * @param metricAggregatorInfos the list of metric aggregator information + * @return the byte size required to store the metric-metric stat pairs + */ + private static long computeMetricEntriesSizeInBytes(List metricAggregatorInfos) { - long metricPairSize = 0; + long totalMetricEntriesSize = 0; for (MetricAggregatorInfo metricAggregatorInfo : metricAggregatorInfos) { - metricPairSize += metricAggregatorInfo.getMetric().getBytes(UTF_8).length; - metricPairSize += metricAggregatorInfo.getMetricStat().getTypeName().getBytes(UTF_8).length; + totalMetricEntriesSize += metricAggregatorInfo.getMetric().getBytes(UTF_8).length; + totalMetricEntriesSize += metricAggregatorInfo.getMetricStat().getTypeName().getBytes(UTF_8).length; } - return metricPairSize; + return totalMetricEntriesSize; } + /** + * Computes the byte size of the star-tree metadata header. + * + * @param compositeFieldType the composite field type of the star-tree field + * @param starTreeFieldName the name of the star-tree field + * @return the byte size of the star-tree metadata header + */ private static int computeHeaderByteSize(CompositeMappedFieldType.CompositeFieldType compositeFieldType, String starTreeFieldName) { // Magic marker (8), version (4), size of header (4) int headerSizeInBytes = 16; @@ -80,6 +113,14 @@ private static int computeHeaderByteSize(CompositeMappedFieldType.CompositeField return headerSizeInBytes; } + /** + * Writes the star-tree metadata header. + * + * @param metaOut the IndexOutput to write the header + * @param compositeFieldType the composite field type of the star-tree field + * @param starTreeFieldName the name of the star-tree field + * @throws IOException if an I/O error occurs while writing the header + */ private static void writeMetaHeader( IndexOutput metaOut, CompositeMappedFieldType.CompositeFieldType compositeFieldType, @@ -98,6 +139,18 @@ private static void writeMetaHeader( metaOut.writeString(compositeFieldType.getName()); } + /** + * Writes the star-tree metadata. + * + * @param metaOut the IndexOutput to write the metadata + * @param writeState the segment write state + * @param metricAggregatorInfos the list of metric aggregator information + * @param starTreeField the star tree field + * @param segmentAggregatedDocCount the aggregated document count for the segment + * @param dataFilePointer the file pointer to the start of the star-tree data + * @param dataFileLength the length of the star-tree data file + * @throws IOException if an I/O error occurs while writing the metadata + */ private static void writeMeta( IndexOutput metaOut, SegmentWriteState writeState, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java index 92930de98970d..c7e8b04d42178 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java @@ -8,6 +8,7 @@ /** * Utility to support Composite Index Star Tree + * * @opensearch.experimental */ package org.opensearch.index.compositeindex.datacube.startree.utils; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java index 59f18efec26b1..9a88f88d9850a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -8,6 +8,7 @@ /** * Core classes for handling composite indices. - * @opensearch.experimental + * + * @opensearch.experimental */ package org.opensearch.index.compositeindex; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index c9f435ccf5e9a..8de583f5aefc5 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -14,15 +14,18 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.composite.Composite90DocValuesFormat; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; @@ -76,9 +79,12 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static List metrics; private static Directory directory; private static FieldInfo[] fieldsInfo; - private static SegmentWriteState state; + private static SegmentWriteState writeState; private static StarTreeField starTreeField; + private static IndexOutput dataOut; + private static IndexOutput metaOut; + @BeforeClass public static void setup() throws IOException { @@ -139,7 +145,21 @@ public static void setup() throws IOException { fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - state = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + + String dataFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite90DocValuesFormat.DATA_EXTENSION + ); + dataOut = writeState.directory.createOutput(dataFileName, writeState.context); + + String metaFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite90DocValuesFormat.META_EXTENSION + ); + metaOut = writeState.directory.createOutput(metaFileName, writeState.context); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -158,7 +178,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(starTreeField, state, mapperService) { + builder = new BaseStarTreeBuilder(metaOut, dataOut, starTreeField, writeState, mapperService) { @Override public void build(List starTreeValuesSubs) throws IOException {} @@ -221,6 +241,8 @@ public void test_reduceStarTreeDocuments() { @Override public void tearDown() throws Exception { super.tearDown(); + dataOut.close(); + metaOut.close(); directory.close(); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index a3924f88ccdfa..7ccb9bcd91c30 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; @@ -22,10 +23,12 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.composite.Composite90DocValuesFormat; import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; @@ -80,6 +83,8 @@ public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { private StarTreeField compositeField; private Map fieldProducerMap; private SegmentWriteState writeState; + private IndexOutput dataOut; + private IndexOutput metaOut; @Before public void setup() throws IOException { @@ -148,6 +153,20 @@ public void setup() throws IOException { FieldInfos fieldInfos = new FieldInfos(fieldsInfo); writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); + String dataFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite90DocValuesFormat.DATA_EXTENSION + ); + dataOut = writeState.directory.createOutput(dataFileName, writeState.context); + + String metaFileName = IndexFileNames.segmentFileName( + writeState.segmentInfo.name, + writeState.segmentSuffix, + Composite90DocValuesFormat.META_EXTENSION + ); + metaOut = writeState.directory.createOutput(metaFileName, writeState.context); + mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); when(mapperService.documentMapper()).thenReturn(documentMapper); @@ -170,7 +189,7 @@ public void setup() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapStarTreeBuilder(compositeField, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); } public void test_sortAndAggregateStarTreeDocuments() throws IOException { @@ -405,7 +424,7 @@ public void test_build_halfFloatMetrics() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapStarTreeBuilder(compositeField, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -500,7 +519,7 @@ public void test_build_floatMetrics() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapStarTreeBuilder(compositeField, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -566,7 +585,7 @@ public void test_build_longMetrics() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapStarTreeBuilder(compositeField, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -699,8 +718,9 @@ public void testMergeFlow() throws IOException { Map f2dimDocIdSetIterators = Map.of("field1", f2d1sndv, "field3", f2d2sndv); Map f2metricDocIdSetIterators = Map.of("field2", f2m1sndv); StarTreeValues starTreeValues2 = new StarTreeValues(sf, null, f2dimDocIdSetIterators, f2metricDocIdSetIterators); - OnHeapStarTreeBuilder builder = new OnHeapStarTreeBuilder(sf, writeState, mapperService); + OnHeapStarTreeBuilder builder = new OnHeapStarTreeBuilder(metaOut, dataOut, sf, writeState, mapperService); Iterator starTreeDocumentIterator = builder.mergeStarTrees(List.of(starTreeValues, starTreeValues2)); + /** * Asserting following dim / metrics [ dim1, dim2 / Sum [ metric] ] * [0, 0] | [0.0] @@ -710,6 +730,7 @@ public void testMergeFlow() throws IOException { * [5, 5] | [100.0] * [null, 2] | [40.0] */ + while (starTreeDocumentIterator.hasNext()) { StarTreeDocument starTreeDocument = starTreeDocumentIterator.next(); assertEquals( @@ -774,6 +795,8 @@ public long cost() { @Override public void tearDown() throws Exception { super.tearDown(); + dataOut.close(); + metaOut.close(); directory.close(); } }