Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Star tree] Moving compositeCodec to composite912Codec and dropping composite99Codec #16227

Merged
merged 3 commits into from
Oct 9, 2024
Merged

[Star tree] Moving compositeCodec to composite912Codec and dropping composite99Codec #16227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c001ee
Refactoring builder tests
bharath-techie Sep 19, 2024
009d038
composite912 codec refactor changes
bharath-techie Oct 7, 2024
64d6c32
Merge branch 'main' of github.com:opensearch-project/OpenSearch into …
bharath-techie Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.composite.composite912.Composite912Codec;
import org.opensearch.index.codec.composite.composite99.Composite99Codec;
import org.opensearch.index.mapper.MapperService;

import java.util.HashMap;
Expand All @@ -33,7 +32,7 @@
public class CompositeCodecFactory {

// we can use this to track the latest composite codec
public static final String COMPOSITE_CODEC = Composite99Codec.COMPOSITE_INDEX_CODEC_NAME;
public static final String COMPOSITE_CODEC = Composite912Codec.COMPOSITE_INDEX_CODEC_NAME;

public CompositeCodecFactory() {}

Expand Down
4 changes: 2 additions & 2 deletions ...er/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper;
import org.apache.lucene.index.SegmentReadState;
import org.opensearch.index.codec.composite.composite99.Composite99Codec;
import org.opensearch.index.codec.composite.composite912.Composite912Codec;

import java.io.IOException;

Expand Down Expand Up @@ -39,7 +39,7 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec(
) throws IOException {

switch (compositeCodec) {
case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME:
case Composite912Codec.COMPOSITE_INDEX_CODEC_NAME:
Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
state,
dataCodec,
Expand Down
3 changes: 1 addition & 2 deletions ...er/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat;
import org.opensearch.index.mapper.MapperService;

/**
Expand Down Expand Up @@ -53,6 +52,6 @@ protected Composite912Codec(String name, Codec delegate, MapperService mapperSer

@Override
public DocValuesFormat docValuesFormat() {
return new Composite99DocValuesFormat(mapperService);
return new Composite912DocValuesFormat(mapperService);
}
}
22 changes: 11 additions & 11 deletions ...mposite99/Composite99DocValuesFormat.java → ...osite912/Composite912DocValuesFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* compatible open source license.
*/

package org.opensearch.index.codec.composite.composite99;
package org.opensearch.index.codec.composite.composite912;

import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
Expand All @@ -25,7 +25,7 @@
* @opensearch.experimental
*/
@ExperimentalApi
public class Composite99DocValuesFormat extends DocValuesFormat {
public class Composite912DocValuesFormat extends DocValuesFormat {
/**
* Creates a new docvalues format.
*
Expand All @@ -38,10 +38,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
private final MapperService mapperService;

/** Data codec name for Composite Doc Values Format */
public static final String DATA_CODEC_NAME = "Composite99FormatData";
public static final String DATA_CODEC_NAME = "Composite912FormatData";

/** Meta codec name for Composite Doc Values Format */
public static final String META_CODEC_NAME = "Composite99FormatMeta";
public static final String META_CODEC_NAME = "Composite912FormatMeta";

/** Filename extension for the composite index data */
public static final String DATA_EXTENSION = "cid";
Expand All @@ -50,10 +50,10 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
public static final String META_EXTENSION = "cim";

/** Data doc values codec name for Composite Doc Values Format */
public static final String DATA_DOC_VALUES_CODEC = "Composite99DocValuesData";
public static final String DATA_DOC_VALUES_CODEC = "Composite912DocValuesData";

/** Meta doc values codec name for Composite Doc Values Format */
public static final String META_DOC_VALUES_CODEC = "Composite99DocValuesMetadata";
public static final String META_DOC_VALUES_CODEC = "Composite912DocValuesMetadata";

/** Filename extension for the composite index data doc values */
public static final String DATA_DOC_VALUES_EXTENSION = "cidvd";
Expand All @@ -68,27 +68,27 @@ public class Composite99DocValuesFormat extends DocValuesFormat {
public static final int VERSION_CURRENT = VERSION_START;

// needed for SPI
public Composite99DocValuesFormat() {
public Composite912DocValuesFormat() {
this(new Lucene90DocValuesFormat(), null);
}

public Composite99DocValuesFormat(MapperService mapperService) {
public Composite912DocValuesFormat(MapperService mapperService) {
this(new Lucene90DocValuesFormat(), mapperService);
}

public Composite99DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) {
public Composite912DocValuesFormat(DocValuesFormat delegate, MapperService mapperService) {
super(delegate.getName());
this.delegate = delegate;
this.mapperService = mapperService;
}

@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new Composite99DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService);
return new Composite912DocValuesWriter(delegate.fieldsConsumer(state), state, mapperService);
}

@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
return new Composite99DocValuesReader(delegate.fieldsProducer(state), state);
return new Composite912DocValuesReader(delegate.fieldsProducer(state), state);
}
}
34 changes: 17 additions & 17 deletions ...mposite99/Composite99DocValuesReader.java → ...osite912/Composite912DocValuesReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* compatible open source license.
*/

package org.opensearch.index.codec.composite.composite99;
package org.opensearch.index.codec.composite.composite912;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -56,8 +56,8 @@
* @opensearch.experimental
*/
@ExperimentalApi
public class Composite99DocValuesReader extends DocValuesProducer implements CompositeIndexReader {
private static final Logger logger = LogManager.getLogger(Composite99DocValuesReader.class);
public class Composite912DocValuesReader extends DocValuesProducer implements CompositeIndexReader {
private static final Logger logger = LogManager.getLogger(Composite912DocValuesReader.class);

private final DocValuesProducer delegate;
private IndexInput dataIn;
Expand All @@ -69,20 +69,20 @@ public class Composite99DocValuesReader extends DocValuesProducer implements Com
private final List<CompositeIndexFieldInfo> compositeFieldInfos = new ArrayList<>();
private SegmentReadState readState;

public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException {
public Composite912DocValuesReader(DocValuesProducer producer, SegmentReadState readState) throws IOException {
this.delegate = producer;
this.fields = new ArrayList<>();

String metaFileName = IndexFileNames.segmentFileName(
readState.segmentInfo.name,
readState.segmentSuffix,
Composite99DocValuesFormat.META_EXTENSION
Composite912DocValuesFormat.META_EXTENSION
);

String dataFileName = IndexFileNames.segmentFileName(
readState.segmentInfo.name,
readState.segmentSuffix,
Composite99DocValuesFormat.DATA_EXTENSION
Composite912DocValuesFormat.DATA_EXTENSION
);

boolean success = false;
Expand All @@ -92,9 +92,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
dataIn = readState.directory.openInput(dataFileName, readState.context);
CodecUtil.checkIndexHeader(
dataIn,
Composite99DocValuesFormat.DATA_CODEC_NAME,
Composite99DocValuesFormat.VERSION_START,
Composite99DocValuesFormat.VERSION_CURRENT,
Composite912DocValuesFormat.DATA_CODEC_NAME,
Composite912DocValuesFormat.VERSION_START,
Composite912DocValuesFormat.VERSION_CURRENT,
readState.segmentInfo.getId(),
readState.segmentSuffix
);
Expand All @@ -105,9 +105,9 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
try {
CodecUtil.checkIndexHeader(
metaIn,
Composite99DocValuesFormat.META_CODEC_NAME,
Composite99DocValuesFormat.VERSION_START,
Composite99DocValuesFormat.VERSION_CURRENT,
Composite912DocValuesFormat.META_CODEC_NAME,
Composite912DocValuesFormat.VERSION_START,
Composite912DocValuesFormat.VERSION_CURRENT,
readState.segmentInfo.getId(),
readState.segmentSuffix
);
Expand Down Expand Up @@ -190,12 +190,12 @@ public Composite99DocValuesReader(DocValuesProducer producer, SegmentReadState r
// initialize star-tree doc values producer

compositeDocValuesProducer = LuceneDocValuesProducerFactory.getDocValuesProducerForCompositeCodec(
Composite99Codec.COMPOSITE_INDEX_CODEC_NAME,
Composite912Codec.COMPOSITE_INDEX_CODEC_NAME,
this.readState,
Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC,
Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
Composite99DocValuesFormat.META_DOC_VALUES_CODEC,
Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION
Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC,
Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
Composite912DocValuesFormat.META_DOC_VALUES_CODEC,
Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION
);

} catch (Throwable t) {
Expand Down
40 changes: 20 additions & 20 deletions ...mposite99/Composite99DocValuesWriter.java → ...osite912/Composite912DocValuesWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* compatible open source license.
*/

package org.opensearch.index.codec.composite.composite99;
package org.opensearch.index.codec.composite.composite912;

import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
Expand Down Expand Up @@ -55,14 +55,14 @@
* @opensearch.experimental
*/
@ExperimentalApi
public class Composite99DocValuesWriter extends DocValuesConsumer {
public class Composite912DocValuesWriter extends DocValuesConsumer {
private final DocValuesConsumer delegate;
private final SegmentWriteState state;
private final MapperService mapperService;
AtomicReference<MergeState> mergeState = new AtomicReference<>();
private final Set<CompositeMappedFieldType> compositeMappedFieldTypes;
private final Set<String> compositeFieldSet;
private DocValuesConsumer composite99DocValuesConsumer;
private DocValuesConsumer compositeDocValuesConsumer;

public IndexOutput dataOut;
public IndexOutput metaOut;
Expand All @@ -72,7 +72,7 @@

private final Map<String, DocValuesProducer> fieldProducerMap = new HashMap<>();

public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService)
public Composite912DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService)
throws IOException {

this.delegate = delegate;
Expand Down Expand Up @@ -101,38 +101,38 @@
// so that all the fields are sparse numeric doc values and not dense numeric doc values
SegmentWriteState consumerWriteState = getSegmentWriteState(segmentWriteState);

this.composite99DocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec(
this.compositeDocValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec(
consumerWriteState,
Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC,
Composite99DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
Composite99DocValuesFormat.META_DOC_VALUES_CODEC,
Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION
Composite912DocValuesFormat.DATA_DOC_VALUES_CODEC,
Composite912DocValuesFormat.DATA_DOC_VALUES_EXTENSION,
Composite912DocValuesFormat.META_DOC_VALUES_CODEC,
Composite912DocValuesFormat.META_DOC_VALUES_EXTENSION
);

String dataFileName = IndexFileNames.segmentFileName(
this.state.segmentInfo.name,
this.state.segmentSuffix,
Composite99DocValuesFormat.DATA_EXTENSION
Composite912DocValuesFormat.DATA_EXTENSION
);
dataOut = this.state.directory.createOutput(dataFileName, this.state.context);
CodecUtil.writeIndexHeader(
dataOut,
Composite99DocValuesFormat.DATA_CODEC_NAME,
Composite99DocValuesFormat.VERSION_CURRENT,
Composite912DocValuesFormat.DATA_CODEC_NAME,
Composite912DocValuesFormat.VERSION_CURRENT,
this.state.segmentInfo.getId(),
this.state.segmentSuffix
);

String metaFileName = IndexFileNames.segmentFileName(
this.state.segmentInfo.name,
this.state.segmentSuffix,
Composite99DocValuesFormat.META_EXTENSION
Composite912DocValuesFormat.META_EXTENSION
);
metaOut = this.state.directory.createOutput(metaFileName, this.state.context);
CodecUtil.writeIndexHeader(
metaOut,
Composite99DocValuesFormat.META_CODEC_NAME,
Composite99DocValuesFormat.VERSION_CURRENT,
Composite912DocValuesFormat.META_CODEC_NAME,
Composite912DocValuesFormat.VERSION_CURRENT,
this.state.segmentInfo.getId(),
this.state.segmentSuffix
);
Expand Down Expand Up @@ -197,12 +197,12 @@
success = true;
} finally {
if (success) {
IOUtils.close(dataOut, metaOut, composite99DocValuesConsumer);
IOUtils.close(dataOut, metaOut, compositeDocValuesConsumer);
} else {
IOUtils.closeWhileHandlingException(dataOut, metaOut, composite99DocValuesConsumer);
IOUtils.closeWhileHandlingException(dataOut, metaOut, compositeDocValuesConsumer);

Check warning on line 202 in server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesWriter.java#L202 

Added line #L202 was not covered by tests
}
metaOut = dataOut = null;
composite99DocValuesConsumer = null;
compositeDocValuesConsumer = null;
}
}

Expand All @@ -224,7 +224,7 @@
for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) {
if (mappedType instanceof StarTreeMapper.StarTreeFieldType) {
try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) {
starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, composite99DocValuesConsumer);
starTreesBuilder.build(metaOut, dataOut, fieldProducerMap, compositeDocValuesConsumer);
}
}
}
Expand Down Expand Up @@ -313,7 +313,7 @@
}
}
try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService, fieldNumberAcrossCompositeFields)) {
starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeSubsPerField, composite99DocValuesConsumer);
starTreesBuilder.buildDuringMerge(metaOut, dataOut, starTreeSubsPerField, compositeDocValuesConsumer);
}
}

Expand Down
51 changes: 0 additions & 51 deletions server/src/main/java/org/opensearch/index/codec/composite/composite99/Composite99Codec.java
View file
Open in desktop

This file was deleted.

12 changes: 0 additions & 12 deletions server/src/main/java/org/opensearch/index/codec/composite/composite99/package-info.java
View file
Open in desktop

This file was deleted.

Loading
Loading