segment info changes

Signed-off-by: Bharathwaj G <[email protected]>
apache · May 3, 2024 · 75618e3 · 75618e3
1 parent e1bd26f
commit 75618e3
Show file tree

Hide file tree

Showing 24 changed files with 100 additions and 65 deletions.
diff --git a/...codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java b/...codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java
@@ -312,7 +312,8 @@ private SegmentInfo parseSegmentInfo(
             diagnostics,
             segmentID,
             attributes,
-            indexSort);
+            indexSort,
+            null);
     si.setFiles(files);
     return si;
   }

diff --git a/...codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java b/...codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java
@@ -169,7 +169,8 @@ private SegmentInfo parseSegmentInfo(
             diagnostics,
             segmentID,
             attributes,
-            indexSort);
+            indexSort,
+            null);
     si.setFiles(files);
     return si;
   }

diff --git a/...codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SegmentInfoFormat.java b/...codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SegmentInfoFormat.java
@@ -167,7 +167,8 @@ private SegmentInfo parseSegmentInfo(
             diagnostics,
             segmentID,
             attributes,
-            indexSort);
+            indexSort,
+            null);
     si.setFiles(files);
     return si;
   }

diff --git a/...decs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWSegmentInfoFormat.java b/...decs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWSegmentInfoFormat.java
@@ -158,7 +158,8 @@ private SegmentInfo parseSegmentInfo(
             diagnostics,
             segmentID,
             attributes,
-            indexSort);
+            indexSort,
+            null);
     si.setFiles(files);
     return si;
   }

diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
@@ -220,7 +220,8 @@ public SegmentInfo read(
               diagnostics,
               id,
               attributes,
-              indexSort);
+              indexSort,
+              null); // TODO
       info.setFiles(files);
       return info;
     }

diff --git a/.../codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextSegmentInfoFormat.java b/.../codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextSegmentInfoFormat.java
@@ -61,7 +61,8 @@ public void testFileIsUTF8() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.emptyMap(),
-            null);
+            null,
+            null); // TODO
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
     String segFileName =

diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SegmentInfoFormat.java
@@ -150,22 +150,21 @@ private SegmentInfo parseSegmentInfo(
     }
 
     // TODO : bwc tests are failing , need to check how to handle this
-    DataCubesConfig compositeConfig = null;
-    //    int numCompositeFields = input.readVInt();
-    //    DataCubesConfig compositeConfig;
-    //    if (numCompositeFields > 0) {
-    //      DataCubeField[] compositeIndexFields = new DataCubeField[numCompositeFields];
-    //      for (int i = 0; i < numCompositeFields; i++) {
+    DataCubesConfig dataCubesConfig = null;
+    //    int numDataCubeFields = input.readVInt();
+    //    DataCubesConfig dataCubesConfig;
+    //    if (numDataCubeFields > 0) {
+    //      DataCubeField[] dataCubeFields = new DataCubeField[numDataCubeFields];
+    //      for (int i = 0; i < numDataCubeFields; i++) {
     //        String name = input.readString();
-    //        compositeIndexFields[i] =
-    // DataCubeFieldProvider.forName(name).readDataCubeField(input);
+    //        dataCubeFields[i] = DataCubeFieldProvider.forName(name).readDataCubeField(input);
     //      }
-    //      compositeConfig = new DataCubesConfig(compositeIndexFields);
-    //    } else if (numCompositeFields < 0) {
+    //      dataCubesConfig = new DataCubesConfig(dataCubeFields);
+    //    } else if (numDataCubeFields < 0) {
     //      throw new CorruptIndexException(
-    //          "invalid composite field count: " + numCompositeFields, input);
+    //          "invalid data cubes field count: " + numDataCubeFields, input);
     //    } else {
-    //      compositeConfig = null;
+    //      dataCubesConfig = null;
     //    }
 
     SegmentInfo si =
@@ -182,7 +181,7 @@ private SegmentInfo parseSegmentInfo(
             segmentID,
             attributes,
             indexSort,
-            compositeConfig);
+            dataCubesConfig);
     si.setFiles(files);
     return si;
   }
@@ -252,14 +251,15 @@ private void writeSegmentInfo(DataOutput output, SegmentInfo si) throws IOExcept
       output.writeString(sorter.getProviderName());
       SortFieldProvider.write(sortField, output);
     }
+
     // TODO
-    //    DataCubesConfig compositeConfig = si.getDataCubesConfig();
-    //    int numCompositeFields = compositeConfig == null ? 0 : compositeConfig.getFields().length;
-    //    output.writeVInt(numCompositeFields);
-    //    for (int i = 0; i < numCompositeFields; ++i) {
-    //      DataCubeField compositeIndexField = compositeConfig.getFields()[i];
-    //      output.writeString(compositeIndexField.getProviderName());
-    //      DataCubeFieldProvider.write(compositeIndexField, output);
+    //    DataCubesConfig dataCubesConfig = si.getDataCubesConfig();
+    //    int numDataCubeFields = dataCubesConfig == null ? 0 : dataCubesConfig.getFields().length;
+    //    output.writeVInt(numDataCubeFields);
+    //    for (int i = 0; i < numDataCubeFields; ++i) {
+    //      DataCubeField dataCubeField = dataCubesConfig.getFields()[i];
+    //      output.writeString(dataCubeField.getProviderName());
+    //      DataCubeFieldProvider.write(dataCubeField, output);
     //    }
   }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
@@ -146,36 +146,6 @@ public SegmentInfo(
     this.dataCubesConfig = dataCubesConfig;
   }
 
-  /** TODO: remove this? */
-  public SegmentInfo(
-      Directory dir,
-      Version version,
-      Version minVersion,
-      String name,
-      int maxDoc,
-      boolean isCompoundFile,
-      boolean hasBlocks,
-      Codec codec,
-      Map<String, String> diagnostics,
-      byte[] id,
-      Map<String, String> attributes,
-      Sort indexSort) {
-    this(
-        dir,
-        version,
-        minVersion,
-        name,
-        maxDoc,
-        isCompoundFile,
-        hasBlocks,
-        codec,
-        diagnostics,
-        id,
-        attributes,
-        indexSort,
-        null);
-  }
-
   /**
    * Mark whether this segment is stored as a compound file.
    *

diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
@@ -137,6 +137,7 @@ MergeState merge() throws IOException {
 
     TaskExecutor taskExecutor = new TaskExecutor(mergeState.intraMergeTaskExecutor);
     List<Callable<Void>> mergingTasks = new ArrayList<>();
+
     mergingTasks.add(
         () -> {
           if (mergeState.mergeFieldInfos.hasNorms()) {
@@ -183,6 +184,14 @@ MergeState merge() throws IOException {
             return null;
           });
     }
+    if (mergeState.segmentInfo.getDataCubesConfig() != null) {
+      mergingTasks.add(
+          () -> {
+            mergeWithLogging(
+                this::mergeDataCubes, segmentWriteState, segmentReadState, "data cubes", numMerged);
+            return null;
+          });
+    }
 
     taskExecutor.invokeAll(mergingTasks);
     // write the merged infos
@@ -276,6 +285,16 @@ private int mergeTermVectors() throws IOException {
     }
   }
 
+  private void mergeDataCubes(
+      SegmentWriteState segmentWriteState, SegmentReadState segmentReadState) throws IOException {
+    try (DataCubesDocValuesConsumer consumer =
+        codec
+            .dataCubesFormat()
+            .fieldsConsumer(segmentWriteState, mergeState.segmentInfo.getDataCubesConfig())) {
+      consumer.merge(mergeState);
+    }
+  }
+
   private void mergeVectorValues(
       SegmentWriteState segmentWriteState, SegmentReadState segmentReadState) throws IOException {
     try (KnnVectorsWriter writer = codec.knnVectorsFormat().fieldsWriter(segmentWriteState)) {

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
@@ -251,6 +251,7 @@ public void testFixedPostings() throws Throwable {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
 
     this.write(si, fieldInfos, dir, fields);
@@ -331,6 +332,7 @@ public void testRandomPostings() throws Throwable {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
     this.write(si, fieldInfos, dir, fields);
 

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
@@ -463,6 +463,7 @@ public void merge(MergePolicy.OneMerge merge) throws IOException {
                           Collections.emptyMap(),
                           StringHelper.randomId(),
                           new HashMap<>(),
+                          null,
                           null),
                       0,
                       0,

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -229,6 +229,7 @@ private SegmentCommitInfo merge(
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
 
     SegmentMerger merger =

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestMergePolicy.java
@@ -158,7 +158,8 @@ private static MergePolicy.MergeSpecification createRandomMergeSpecification(
                       random(), StringHelper.ID_LENGTH, StringHelper.ID_LENGTH)
                   .getBytes(StandardCharsets.US_ASCII),
               Collections.emptyMap(), // attributes
-              null /* indexSort */);
+              null /* indexSort */,
+              null /*datacubesconfig*/);
       final List<SegmentCommitInfo> segments = new LinkedList<SegmentCommitInfo>();
       segments.add(new SegmentCommitInfo(si, 0, 0, 0, 0, 0, StringHelper.randomId()));
       ms.add(new MergePolicy.OneMerge(segments));

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestOneMergeWrappingMergePolicy.java
@@ -138,7 +138,8 @@ private static MergePolicy.MergeSpecification createRandomMergeSpecification(Dir
                         random(), StringHelper.ID_LENGTH, StringHelper.ID_LENGTH)
                     .getBytes(StandardCharsets.US_ASCII),
                 Collections.emptyMap(), // attributes
-                null /* indexSort */);
+                null /* indexSort */,
+                null /*datacubesconfig*/);
         final List<SegmentCommitInfo> segments = new LinkedList<SegmentCommitInfo>();
         segments.add(new SegmentCommitInfo(si, 0, 0, 0, 0, 0, StringHelper.randomId()));
         ms.add(new MergePolicy.OneMerge(segments));

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingDeletes.java
@@ -51,6 +51,7 @@ public void testDeleteDoc() throws IOException {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
     SegmentCommitInfo commitInfo =
         new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
@@ -98,6 +99,7 @@ public void testWriteLiveDocs() throws IOException {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
     SegmentCommitInfo commitInfo =
         new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentInfos.java
@@ -81,6 +81,7 @@ public void testVersionsOneSegment() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.<String, String>emptyMap(),
+            null,
             null);
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -116,6 +117,7 @@ public void testVersionsTwoSegments() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.<String, String>emptyMap(),
+            null,
             null);
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -136,6 +138,7 @@ public void testVersionsTwoSegments() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.<String, String>emptyMap(),
+            null,
             null);
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -181,7 +184,8 @@ public void testToString() throws Throwable {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null);
     assertEquals(
         "TEST(" + Version.LATEST.toString() + ")" + ":C10000" + ":[indexSort=<doc>]",
         si.toString());
@@ -200,7 +204,8 @@ public void testToString() throws Throwable {
             diagnostics,
             StringHelper.randomId(),
             new HashMap<>(),
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null); // TODO
     assertEquals(
         "TEST("
             + Version.LATEST.toString()
@@ -226,7 +231,8 @@ public void testToString() throws Throwable {
             Collections.emptyMap(),
             StringHelper.randomId(),
             attributes,
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null);
     assertEquals(
         "TEST("
             + Version.LATEST.toString()
@@ -252,7 +258,8 @@ public void testToString() throws Throwable {
             diagnostics,
             StringHelper.randomId(),
             attributes,
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null);
     assertEquals(
         "TEST("
             + Version.LATEST.toString()
@@ -287,6 +294,7 @@ public void testIDChangesOnAdvance() throws IOException {
               Collections.<String, String>emptyMap(),
               StringHelper.randomId(),
               Collections.<String, String>emptyMap(),
+              null,
               null);
       SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, id);
       assertEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
@@ -334,6 +342,7 @@ public void testBitFlippedTriggersCorruptIndexException() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.<String, String>emptyMap(),
+            null,
             null);
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -354,6 +363,7 @@ public void testBitFlippedTriggersCorruptIndexException() throws IOException {
             Collections.<String, String>emptyMap(),
             id,
             Collections.<String, String>emptyMap(),
+            null,
             null);
     info.setFiles(Collections.<String>emptySet());
     codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -426,7 +436,8 @@ public void testAddDiagnostics() throws Throwable {
             diagnostics,
             StringHelper.randomId(),
             new HashMap<>(),
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null);
     si.addDiagnostics(Map.of("key3", "value3"));
     assertEquals(Map.of("key1", "value1", "key2", "value2", "key3", "value3"), si.getDiagnostics());
 
@@ -444,7 +455,8 @@ public void testAddDiagnostics() throws Throwable {
             diagnostics,
             StringHelper.randomId(),
             new HashMap<>(),
-            Sort.INDEXORDER);
+            Sort.INDEXORDER,
+            null);
     si.addDiagnostics(Map.of("key2", "foo"));
     assertEquals(Map.of("key1", "value1", "key2", "foo"), si.getDiagnostics());
 

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
@@ -97,6 +97,7 @@ public void testMerge() throws IOException {
             Collections.emptyMap(),
             StringHelper.randomId(),
             new HashMap<>(),
+            null,
             null);
 
     SegmentMerger merger =

diff --git a/lucene/misc/src/java/org/apache/lucene/misc/index/IndexSplitter.java b/lucene/misc/src/java/org/apache/lucene/misc/index/IndexSplitter.java
@@ -128,6 +128,7 @@ public void split(Path destDir, String[] segs) throws IOException {
               info.getDiagnostics(),
               info.getId(),
               Collections.emptyMap(),
+              null,
               null);
       destInfos.add(
           new SegmentCommitInfo(
-Original file line number
+Diff line change
@@ Expand Up @@
                               Collections.emptyMap(),
                               StringHelper.randomId(),
                               new HashMap<>(),
+                              null,
                               null),
 ,
 ,
@@ Expand Down @@