Merge branch 'main' into java_21

buildSrc/src/main/java/org/apache/lucene/gradle/datasets/ExtractReuters.java

@@ -27,6 +27,7 @@
 import java.nio.file.StandardCopyOption;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 /**
  * Split the Reuters SGML documents into Simple Text files containing:
@@ -44,9 +45,10 @@ public ExtractReuters(Path reutersDir, Path outputDir) throws IOException {
   public void extract() throws IOException {
     long count = 0;
     Files.createDirectories(outputDir);
-
-    if (Files.list(outputDir).count() > 0) {
-      throw new IOException("The output directory must be empty: " + outputDir);
+    try(Stream<Path> files = Files.list(outputDir)) {
+      if (files.count() > 0) {
+        throw new IOException("The output directory must be empty: " + outputDir);
+      }
     }
 
     try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) {

dev-tools/scripts/releaseWizard.py

@@ -63,7 +63,6 @@
 import scriptutil
 from consolemenu import ConsoleMenu
 from consolemenu.items import FunctionItem, SubmenuItem, ExitItem
-from consolemenu.screen import Screen
 from scriptutil import BranchType, Version, download, run
 
 # Lucene-to-Java version mapping
@@ -654,16 +653,16 @@ def get_title(self):
         return "%s%s (%d/%d)" % (prefix, self.title, self.num_done(), self.num_applies())
 
     def get_submenu(self):
-        menu = UpdatableConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(),
-                           screen=MyScreen())
+        menu = ConsoleMenu(title=self.title, subtitle=self.get_subtitle, prologue_text=self.get_description(),
+                           clear_screen=False)
         menu.exit_item = CustomExitItem("Return")
         for todo in self.get_todos():
             if todo.applies(state.release_type):
                 menu.append_item(todo.get_menu_item())
         return menu
 
     def get_menu_item(self):
-        item = UpdatableSubmenuItem(self.get_title, self.get_submenu())
+        item = SubmenuItem(self.get_title, self.get_submenu())
         return item
 
     def get_todos(self):
@@ -820,7 +819,7 @@ def display_and_confirm(self):
             print("ERROR while executing todo %s (%s)" % (self.get_title(), e))
 
     def get_menu_item(self):
-        return UpdatableFunctionItem(self.get_title, self.display_and_confirm)
+        return FunctionItem(self.get_title, self.display_and_confirm)
 
     def clone(self):
         clone = Todo(self.id, self.title, description=self.description)
@@ -1234,104 +1233,6 @@ def pause(fun=None):
     input("\nPress ENTER to continue...")
 
 
-# Custom classes for ConsoleMenu, to make menu texts dynamic
-# Needed until https://github.com/aegirhall/console-menu/pull/25 is released
-# See https://pypi.org/project/console-menu/ for other docs
-
-class UpdatableConsoleMenu(ConsoleMenu):
-
-    def __repr__(self):
-        return "%s: %s. %d items" % (self.get_title(), self.get_subtitle(), len(self.items))
-
-    def draw(self):
-        """
-        Refreshes the screen and redraws the menu. Should be called whenever something changes that needs to be redrawn.
-        """
-        self.screen.printf(self.formatter.format(title=self.get_title(), subtitle=self.get_subtitle(), items=self.items,
-                                                 prologue_text=self.get_prologue_text(), epilogue_text=self.get_epilogue_text()))
-
-    # Getters to get text in case method reference
-    def get_title(self):
-        return self.title() if callable(self.title) else self.title
-
-    def get_subtitle(self):
-        return self.subtitle() if callable(self.subtitle) else self.subtitle
-
-    def get_prologue_text(self):
-        return self.prologue_text() if callable(self.prologue_text) else self.prologue_text
-
-    def get_epilogue_text(self):
-        return self.epilogue_text() if callable(self.epilogue_text) else self.epilogue_text
-
-
-class UpdatableSubmenuItem(SubmenuItem):
-    def __init__(self, text, submenu, menu=None, should_exit=False):
-        """
-        :ivar ConsoleMenu self.submenu: The submenu to be opened when this item is selected
-        """
-        super(UpdatableSubmenuItem, self).__init__(text=text, menu=menu, should_exit=should_exit, submenu=submenu)
-
-        if menu:
-            self.get_submenu().parent = menu
-
-    def show(self, index):
-        return "%2d - %s" % (index + 1, self.get_text())
-
-    # Getters to get text in case method reference
-    def get_text(self):
-        return self.text() if callable(self.text) else self.text
-
-    def set_menu(self, menu):
-        """
-        Sets the menu of this item.
-        Should be used instead of directly accessing the menu attribute for this class.
-
-        :param ConsoleMenu menu: the menu
-        """
-        self.menu = menu
-        self.get_submenu().parent = menu
-
-    def action(self):
-        """
-        This class overrides this method
-        """
-        self.get_submenu().start()
-
-    def clean_up(self):
-        """
-        This class overrides this method
-        """
-        self.get_submenu().join()
-        self.menu.clear_screen()
-        self.menu.resume()
-
-    def get_return(self):
-        """
-        :return: The returned value in the submenu
-        """
-        return self.get_submenu().returned_value
-
-    def get_submenu(self):
-        """
-        We unwrap the submenu variable in case it is a reference to a method that returns a submenu
-        """
-        return self.submenu if not callable(self.submenu) else self.submenu()
-
-
-class UpdatableFunctionItem(FunctionItem):
-    def show(self, index):
-        return "%2d - %s" % (index + 1, self.get_text())
-
-    # Getters to get text in case method reference
-    def get_text(self):
-        return self.text() if callable(self.text) else self.text
-
-
-class MyScreen(Screen):
-    def clear(self):
-        return
-
-
 class CustomExitItem(ExitItem):
     def show(self, index):
         return super(CustomExitItem, self).show(index)
@@ -1346,6 +1247,13 @@ def main():
     global templates
 
     print("Lucene releaseWizard v%s" % getScriptVersion())
+
+    try:
+      ConsoleMenu(clear_screen=True)
+    except Exception as e:
+      sys.exit("You need to install 'consolemenu' package version 0.7.1 for the Wizard to function. Please run 'pip "
+               "install -r requirements.txt'")
+
     c = parse_config()
 
     if c.dry:
@@ -1402,18 +1310,18 @@ def main():
     lucene_news_file = os.path.join(state.get_website_git_folder(), 'content', 'core', 'core_news',
       "%s-%s-available.md" % (state.get_release_date_iso(), state.release_version.replace(".", "-")))
 
-    main_menu = UpdatableConsoleMenu(title="Lucene ReleaseWizard",
+    main_menu = ConsoleMenu(title="Lucene ReleaseWizard",
                             subtitle=get_releasing_text,
                             prologue_text="Welcome to the release wizard. From here you can manage the process including creating new RCs. "
                                           "All changes are persisted, so you can exit any time and continue later. Make sure to read the Help section.",
                             epilogue_text="® 2022 The Lucene project. Licensed under the Apache License 2.0\nScript version v%s)" % getScriptVersion(),
-                            screen=MyScreen())
+                            clear_screen=False)
 
-    todo_menu = UpdatableConsoleMenu(title=get_releasing_text,
+    todo_menu = ConsoleMenu(title=get_releasing_text,
                             subtitle=get_subtitle,
                             prologue_text=None,
                             epilogue_text=None,
-                            screen=MyScreen())
+                            clear_screen=False)
     todo_menu.exit_item = CustomExitItem("Return")
 
     for todo_group in state.todo_groups:
@@ -1422,14 +1330,14 @@ def main():
             menu_item.set_menu(todo_menu)
             todo_menu.append_item(menu_item)
 
-    main_menu.append_item(UpdatableSubmenuItem(get_todo_menuitem_title, todo_menu, menu=main_menu))
-    main_menu.append_item(UpdatableFunctionItem(get_start_new_rc_menu_title, start_new_rc))
-    main_menu.append_item(UpdatableFunctionItem('Clear and restart current RC', state.clear_rc))
-    main_menu.append_item(UpdatableFunctionItem("Clear all state, restart the %s release" % state.release_version, reset_state))
-    main_menu.append_item(UpdatableFunctionItem('Start release for a different version', release_other_version))
-    main_menu.append_item(UpdatableFunctionItem('Generate Asciidoc guide for this release', generate_asciidoc))
-    # main_menu.append_item(UpdatableFunctionItem('Dump YAML', dump_yaml))
-    main_menu.append_item(UpdatableFunctionItem('Help', help))
+    main_menu.append_item(SubmenuItem(get_todo_menuitem_title, todo_menu, menu=main_menu))
+    main_menu.append_item(FunctionItem(get_start_new_rc_menu_title, start_new_rc))
+    main_menu.append_item(FunctionItem('Clear and restart current RC', state.clear_rc))
+    main_menu.append_item(FunctionItem("Clear all state, restart the %s release" % state.release_version, reset_state))
+    main_menu.append_item(FunctionItem('Start release for a different version', release_other_version))
+    main_menu.append_item(FunctionItem('Generate Asciidoc guide for this release', generate_asciidoc))
+    # main_menu.append_item(FunctionItem('Dump YAML', dump_yaml))
+    main_menu.append_item(FunctionItem('Help', help))
 
     main_menu.show()
 

dev-tools/scripts/releaseWizard.yaml

@@ -521,7 +521,7 @@ groups:
         addition wait a couple more days? Merges of bug fixes into the branch
         may become more difficult.
       * Only Github issues with Milestone {{ release_version_major }}.{{ release_version_minor }}
-        and priority "Blocker" will delay a release candidate build.
+        will delay a release candidate build.
       ----
     types:
     - major
@@ -979,8 +979,8 @@ groups:
     title: Publish docs, changes and javadocs
     description: |
       Ensure your refrigerator has at least 2 beers - the svn import operation can take a while,
-      depending on your upload bandwidth. We'll publish this directly to the production tree.
-      At the end of the task, the two links below shall work.
+      depending on your upload bandwidth. We'll publish this directly to the production tree.  At
+      the end of the task, the two links below shall work.
     links:
     - http://lucene.apache.org/core/{{ version }}
     vars:
@@ -1126,12 +1126,18 @@ groups:
         comment: Push all changes
         logfile: push-website.log
     post_description: |
-      Wait a few minutes for the build to happen. You can follow the site build at https://ci2.apache.org/#/builders/3
-      and view the staged site at https://lucene.staged.apache.org
-      Verify that correct links and versions are mentioned in download pages, download buttons etc.
-      If you find anything wrong, then commit and push any changes and check again.
-
-      Next step is to merge the changes to branch 'production' in order to publish the site.
+      Wait a few minutes for the build to happen. You can follow the site build at
+      https://ci2.apache.org/#/builders/3 and view the staged site at
+      https://lucene.staged.apache.org Verify that correct links and versions are mentioned in
+      download pages, download buttons etc.  If you find anything wrong, then commit and push any
+      changes and check again.  You may find that the publish fails, leaving a directory listing
+      instead a beautiful website. If this happens, check the "builder" link and click through into
+      its details to find possible error messages produced by the website publication process. You
+      may have produced malformed Markdown. Or the website publish may just fail for some reason out
+      of your control. If this happens, you can attempt to retrigger the publishing with some
+      innocuous changes.  Next step is to merge the changes to branch 'production' in order to
+      publish the site. Before doing this, you may want to replenish your stock of beers, or get
+      stronger stuff.
     links:
     - https://ci2.apache.org/#/builders/3
     - https://lucene.staged.apache.org
@@ -1159,7 +1165,8 @@ groups:
     post_description: |
       Wait a few minutes for the build to happen. You can follow the site build at https://ci2.apache.org/#/builders/3
 
-      Verify on https://lucene.apache.org that the site is OK.
+      Verify on https://lucene.apache.org that the site is OK. It really should be, but see staging
+      site publication instructions for possible debugging/recovery options if it is not.
 
       You can now also verify that http://lucene.apache.org/core/api/core/ redirects to the latest version
     links:

dev-tools/scripts/requirements.txt

@@ -1,8 +1,8 @@
-six>=1.11.0
-Jinja2>=2.10.1
-PyYAML>=5.1
-holidays>=0.9.10
-ics>=0.4
-console-menu>=0.5.1
-PyGithub
-jira
+six~=1.16.0
+Jinja2~=3.1.1
+PyYAML~=6.0
+holidays~=0.16
+ics~=0.7.2
+console-menu~=0.7.1
+PyGithub~=1.56
+jira~=3.4.1

gradle/java/modules.gradle

@@ -67,6 +67,12 @@ allprojects {
       tasks.named(sourceSet.getCompileJavaTaskName()).configure({ JavaCompile task ->
         task.dependsOn modularPaths.compileModulePathConfiguration
 
+        // GH-12742: add the modular path as inputs so that if anything changes, the task
+        // is not up to date and is re-run. I [dw] believe this should be a @Classpath parameter
+        // on the task itself... but I don't know how to implement this on an existing class.
+        // this is a workaround but should work just fine though.
+        task.inputs.files(modularPaths.compileModulePathConfiguration)
+
         // LUCENE-10327: don't allow gradle to emit an empty sourcepath as it would break
         // compilation of modules.
         task.options.setSourcepath(sourceSet.java.sourceDirectories)

gradle/testing/randomization.gradle

@@ -67,7 +67,7 @@ allprojects {
           // seed, repetition and amplification.
           [propName: 'tests.seed', value: { -> rootSeed }, description: "Sets the master randomization seed."],
           [propName: 'tests.iters', value: null, description: "Duplicate (re-run) each test case N times."],
-          [propName: 'tests.multiplier', value: 1, description: "Value multiplier for randomized tests."],
+          [propName: 'tests.multiplier', value: null, description: "Value multiplier for randomized tests."],
           [propName: 'tests.maxfailures', value: null, description: "Skip tests after a given number of failures."],
           [propName: 'tests.timeoutSuite', value: null, description: "Timeout (in millis) for an entire suite."],
           [propName: 'tests.failfast', value: "false", description: "Stop the build early on failure.", buildOnly: true],

lucene/CHANGES.txt

@@ -62,9 +62,11 @@ API Changes
 
 * GITHUB#12599: Add RandomAccessInput#readBytes method to the RandomAccessInput interface. (Ignacio Vera)
 
-* GITHUB#12709 Consolidate FSTStore and BytesStore in FST. Created FSTReader which contains the common methods
+* GITHUB#12709: Consolidate FSTStore and BytesStore in FST. Created FSTReader which contains the common methods
   of the two (Anh Dung Bui)
 
+* GITHUB#12735: Remove FSTCompiler#getTermCount() and FSTCompiler.UnCompiledNode#inputCount (Anh Dung Bui)
+
 New Features
 ---------------------
 
@@ -208,6 +210,9 @@ Improvements
 
 * GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)
 
+* GITHUB#12754: Refactor lookup of Hotspot VM options and do not initialize constants with NULL
+  if SecurityManager prevents access.  (Uwe Schindler)
+
 Optimizations
 ---------------------
 * GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
@@ -251,6 +256,11 @@ Optimizations
 * GITHUB#12719: Top-level conjunctions that are not sorted by score now have a
   specialized bulk scorer. (Adrien Grand)
 
+* GITHUB#1052: Faster merging of terms enums. (Adrien Grand)
+
+* GITHUB#11903: Faster sort on high-cardinality string fields. (Adrien Grand)
+
+
 Changes in runtime behavior
 ---------------------
 
@@ -278,7 +288,14 @@ Bug Fixes
 Build
 ---------------------
 
+* GITHUB#12752: tests.multiplier could be omitted in test failure reproduce lines (esp. in
+  nightly mode). (Dawid Weiss)
+
+* GITHUB#12742: JavaCompile tasks may be in up-to-date state when modular dependencies have changed 
+  leading to odd runtime errors (Chris Hostetter, Dawid Weiss)
+
 * GITHUB#12612: Upgrade forbiddenapis to version 3.6 and ASM for APIJAR extraction to 9.6.  (Uwe Schindler)
+
 * GITHUB#12655: Upgrade to Gradle 8.4 (Kevin Risden)
 
 Other

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CSVUtil.java → lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CSVUtil.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.lucene.analysis.ja.dict;
+package org.apache.lucene.analysis.util;
 
 import java.util.ArrayList;
 import java.util.regex.Matcher;
@@ -69,7 +69,7 @@ public static String[] parse(String line) {
       return new String[0];
     }
 
-    return result.toArray(new String[result.size()]);
+    return result.toArray(new String[0]);
   }
 
   private static String unQuoteUnEscape(String original) {
@@ -83,7 +83,7 @@ private static String unQuoteUnEscape(String original) {
       }
 
       // Unescape
-      if (result.indexOf(ESCAPED_QUOTE) >= 0) {
+      if (result.contains(ESCAPED_QUOTE)) {
         result = result.replace(ESCAPED_QUOTE, "\"");
       }
     }