star tree file formats

server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java

@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+public class CompositeIndexConstants {
+
+    public static final long MAGIC_MARKER = 0xC0950513F1E1DL; // OpenSearch Tree
+    public static final int VERSION = 1;
+
+}

server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java

@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.IndexInput;
+import org.opensearch.index.compositeindex.datacube.startree.meta.StarTreeMetadata;
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.opensearch.index.compositeindex.CompositeIndexConstants.MAGIC_MARKER;
+import static org.opensearch.index.compositeindex.CompositeIndexConstants.VERSION;
+
+public class CompositeIndexMetadata {
+
+    private static final Logger logger = LogManager.getLogger(CompositeIndexMetadata.class);
+    private final Map<String, StarTreeMetadata> starTreeMetadataMap = new HashMap<>();
+
+    public CompositeIndexMetadata(IndexInput meta) throws IOException {
+        long magicMarker = meta.readLong();
+        if (MAGIC_MARKER != magicMarker) {
+            logger.error("Invalid composite field magic marker");
+            throw new IOException("Invalid composite field magic marker");
+        }
+        int version = meta.readInt();
+        if (VERSION != version) {
+            logger.error("Invalid composite field version");
+            throw new IOException("Invalid composite field version");
+        }
+
+        String compositeFieldName = meta.readString();
+        String compositeFieldType = meta.readString();
+
+        CompositeMappedFieldType.CompositeFieldType fieldType = CompositeMappedFieldType.CompositeFieldType.fromName(compositeFieldType);
+
+        switch (fieldType) {
+            // support for type of composite fields can be added in the future.
+            case STAR_TREE:
+                starTreeMetadataMap.put(compositeFieldName, new StarTreeMetadata(meta, compositeFieldName, compositeFieldType));
+                break;
+            default:
+                throw new CorruptIndexException("Invalid composite field type present in the file", meta);
+        }
+
+    }
+
+    public Map<String, StarTreeMetadata> getStarTreeMetadataMap() {
+        return starTreeMetadataMap;
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricEntry.java

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.aggregators;
+
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+
+public class MetricEntry {
+
+    private final String metricName;
+    private final MetricStat metricStat;
+
+    public MetricEntry(String metricName, MetricStat metricStat) {
+        this.metricName = metricName;
+        this.metricStat = metricStat;
+    }
+
+    public String getMetricName() {
+        return metricName;
+    }
+
+    public MetricStat getMetricStat() {
+        return metricStat;
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/StarTreeMetadata.java

@@ -0,0 +1,130 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.meta;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.IndexInput;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricEntry;
+import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Off heap implementation of {@link StarTreeNode}
+ */
+public class StarTreeMetadata implements TreeMetadata {
+    private static final Logger logger = LogManager.getLogger(TreeMetadata.class);
+    private final IndexInput meta;
+    private final String starTreeFieldName;
+    private final String starTreeFieldType;
+    private final List<Integer> dimensionOrdinals;
+    private final List<MetricEntry> metricEntries;
+    private final Integer segmentAggregatedDocCount;
+    private final long dataStartFilePointer;
+    private final long dataLength;
+
+    public StarTreeMetadata(IndexInput meta, String compositeFieldName, String compositeFieldType) throws IOException {
+        this.meta = meta;
+        try {
+            this.starTreeFieldName = compositeFieldName;
+            this.starTreeFieldType = compositeFieldType;
+            this.dimensionOrdinals = readStarTreeDimensions();
+            this.metricEntries = readMetricPairs();
+            this.segmentAggregatedDocCount = readSegmentAggregatedDocCount();
+            this.dataStartFilePointer = readDataStartFilePointer();
+            this.dataLength = readDataLength();
+        } catch (Exception e) {
+            logger.error("Unable to read star-tree metadata from the file");
+            throw new CorruptIndexException("Unable to read star-tree metadata from the file", meta);
+        }
+    }
+
+    @Override
+    public int readDimensionsCount() throws IOException {
+        return meta.readInt();
+    }
+
+    @Override
+    public List<Integer> readStarTreeDimensions() throws IOException {
+        int dimensionCount = readDimensionsCount();
+        List<Integer> dimensionOrdinals = new ArrayList<>();
+
+        for (int i = 0; i < dimensionCount; i++) {
+            dimensionOrdinals.add(meta.readInt());
+        }
+
+        return dimensionOrdinals;
+    }
+
+    @Override
+    public int readMetricsCount() throws IOException {
+        return meta.readInt();
+    }
+
+    @Override
+    public List<MetricEntry> readMetricPairs() throws IOException {
+        int metricCount = readMetricsCount();
+        List<MetricEntry> metricEntries = new ArrayList<>();
+
+        for (int i = 0; i < metricCount; i++) {
+            String metricName = meta.readString();
+            String metricStat = meta.readString();
+            metricEntries.add(new MetricEntry(metricName, MetricStat.fromTypeName(metricStat)));
+        }
+
+        return metricEntries;
+    }
+
+    @Override
+    public int readSegmentAggregatedDocCount() throws IOException {
+        return meta.readInt();
+    }
+
+    @Override
+    public long readDataStartFilePointer() throws IOException {
+        return meta.readLong();
+    }
+
+    @Override
+    public long readDataLength() throws IOException {
+        return meta.readLong();
+    }
+
+    public String getStarTreeFieldName() {
+        return starTreeFieldName;
+    }
+
+    public String getStarTreeFieldType() {
+        return starTreeFieldType;
+    }
+
+    public List<Integer> getDimensionOrdinals() {
+        return dimensionOrdinals;
+    }
+
+    public List<MetricEntry> getMetricPairs() {
+        return metricEntries;
+    }
+
+    public Integer getSegmentAggregatedDocCount() {
+        return segmentAggregatedDocCount;
+    }
+
+    public long getDataStartFilePointer() {
+        return dataStartFilePointer;
+    }
+
+    public long getDataLength() {
+        return dataLength;
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/TreeMetadata.java

@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube.startree.meta;
+
+import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricEntry;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface TreeMetadata {
+
+    int readDimensionsCount() throws IOException;
+
+    List<Integer> readStarTreeDimensions() throws IOException;
+
+    int readMetricsCount() throws IOException;
+
+    List<MetricEntry> readMetricPairs() throws IOException;
+
+    int readSegmentAggregatedDocCount() throws IOException;
+
+    long readDataStartFilePointer() throws IOException;
+
+    long readDataLength() throws IOException;
+
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/meta/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Meta package for star tree
+ */
+package org.opensearch.index.compositeindex.datacube.startree.meta;

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/OffHeapStarTree.java

@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.index.compositeindex.datacube.startree.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.opensearch.index.compositeindex.datacube.startree.meta.StarTreeMetadata;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.opensearch.index.compositeindex.CompositeIndexConstants.MAGIC_MARKER;
+import static org.opensearch.index.compositeindex.CompositeIndexConstants.VERSION;
+
+/**
+ * Off heap implementation of star tree.
+ */
+public class OffHeapStarTree implements StarTree {
+    private static final Logger logger = LogManager.getLogger(OffHeapStarTree.class);
+    private final OffHeapStarTreeNode root;
+    private final List<String> dimensionNames = new ArrayList<>();
+    private final Integer numNodes;
+
+    public OffHeapStarTree(IndexInput data, StarTreeMetadata starTreeMetadata) throws IOException {
+        long magicMarker = data.readLong();
+        if (MAGIC_MARKER != magicMarker) {
+            logger.error("Invalid magic marker");
+            throw new IOException("Invalid magic marker");
+        }
+        int version = data.readInt();
+        if (VERSION != version) {
+            logger.error("Invalid star tree version");
+            throw new IOException("Invalid version");
+        }
+        numNodes = data.readInt(); // num nodes
+
+        // should we get start and end file pointer from meta file?
+        RandomAccessInput in = data.randomAccessSlice(
+            starTreeMetadata.getDataStartFilePointer(),
+            starTreeMetadata.getDataStartFilePointer() + starTreeMetadata.getDataLength()
+        );
+        root = new OffHeapStarTreeNode(in, 0);
+    }
+
+    @Override
+    public StarTreeNode getRoot() {
+        return root;
+    }
+
+    @Override
+    public List<String> getDimensionNames() {
+        return dimensionNames;
+    }
+
+    public Integer getNumNodes() {
+        return numNodes;
+    }
+
+}