Merge branch_9x to fs/branch_9x

.git-blame-ignore-revs

@@ -31,3 +31,6 @@ bb872f74f715fa5f161833d91a7491e8b72c6c18
 501aa0228619b06ba9f962168b584b1c6d8ee4b5
 7e628970044e2e79089aebbe506ee74a6e2ed4da
 
+# SOLR-17026 [google java formatter upgrade]
+74f42abb940fd399dc093f36f470a3ad05c7918b
+

NOTICE.txt

@@ -618,6 +618,13 @@ https://github.com/yonik/noggit
 This product includes the Angular UI UI Grid JavaScript library.
 Copyright (c) 2015 the AngularUI Team, http://angular-ui.github.com
 
+=========================================================================
+==         jsSHA notice                                                ==
+=========================================================================
+
+This product includes the jsSHA library.
+Copyright (c) 2008-2023 Brian Turek, 1998-2009 Paul Johnston & Contributors
+https://github.com/Caligatio/jsSHA
 
 =========================================================================
 ==         grpc notice                                                 ==

build.gradle

@@ -19,15 +19,15 @@ import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
 plugins {
-  id "base"
-  id "com.palantir.consistent-versions" version "2.11.0"
-  id "org.owasp.dependencycheck" version "8.0.1"
-  id 'ca.cutterslade.analyze' version "1.9.0"
-  id 'de.thetaphi.forbiddenapis' version '3.5' apply false
-  id "de.undercouch.download" version "5.2.0" apply false
-  id "net.ltgt.errorprone" version "3.0.1" apply false
-  id 'com.diffplug.spotless' version "6.5.2" apply false
-  id 'com.github.node-gradle.node' version '3.4.0' apply false
+  id 'base'
+  id 'com.palantir.consistent-versions' version '2.16.0'
+  id 'org.owasp.dependencycheck' version '8.4.0'
+  id 'ca.cutterslade.analyze' version '1.9.1'
+  id 'de.thetaphi.forbiddenapis' version '3.6' apply false
+  id 'de.undercouch.download' version '5.5.0' apply false
+  id 'net.ltgt.errorprone' version '3.1.0' apply false
+  id 'com.diffplug.spotless' version '6.5.2' apply false
+  id 'com.github.node-gradle.node' version '7.0.1' apply false
 }
 
 apply from: file('gradle/globals.gradle')
@@ -37,7 +37,7 @@ apply from: file('gradle/globals.gradle')
 // Calculate project version:
 version = {
   // Release manager: update base version here after release:
-  String baseVersion = '9.4.0'
+  String baseVersion = '9.5.0'
 
   // On a release explicitly set release version in one go:
   //  -Dversion.release=x.y.z
@@ -127,6 +127,7 @@ apply from: file('gradle/java/javac.gradle')
 apply from: file('gradle/testing/defaults-tests.gradle')
 apply from: file('gradle/testing/randomization.gradle')
 apply from: file('gradle/testing/fail-on-no-tests.gradle')
+apply from: file('gradle/testing/fail-on-unsupported-jdk.gradle')
 apply from: file('gradle/testing/alternative-jdk-support.gradle')
 apply from: file('gradle/java/jar-manifest.gradle')
 

buildSrc/scriptDepVersions.gradle

@@ -21,11 +21,11 @@
 
 ext {
   scriptDepVersions = [
-      "apache-rat": "0.14",
-      "commons-codec": "1.15",
-      "ecj": "3.30.0",
+      "apache-rat": "0.15",
+      "commons-codec": "1.16.0",
+      "ecj": "3.33.0",
       "javacc": "7.0.12",
-      "jgit": "5.13.1.202206130422-r",
-      "flexmark": "0.64.0",
+      "jgit": "6.7.0.202309050840-r",
+      "flexmark": "0.64.8",
   ]
 }

buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -61,8 +61,8 @@ public static void main(String[] args) {
 
   public static void checkVersion() {
     int major = Runtime.getRuntime().version().feature();
-    if (major < 11 || major > 19) {
-      throw new IllegalStateException("java version must be between 11 and 19, your version: " + major);
+    if (major < 11 || major > 21) {
+      throw new IllegalStateException("java version must be between 11 and 21, your version: " + major);
     }
   }
 

dev-docs/apis.adoc

@@ -0,0 +1,83 @@
+= APIs in Solr
+:toc: left
+
+Solr's codebase currently has a handful of ways of defining APIs.
+This complexity stems largely from two ongoing transitions:
+1. Away from our "v1" APIs and towards the new "v2" API. 
+2. Away from a legacy API framework towards the off-the-shelf JAX-RS library for implementing our v2 APIs
+
+As we finish these transitions, this complexity should simplify considerably.
+But in the interim, this document can help guide developers who need to understand or modify APIs in Solr.
+
+== API Types
+
+APIs in Solr (regardless of whether v1 or v2) can typically be classified as either "per-core" or "cluster" APIs.
+
+Per-core APIs, as the name suggests, typically affect only a single core or collection, usually used to search or analyze that core's contents in some way.
+Implementation-wise, they're registered on the `SolrCore` object itself.
+They are configured in solrconfig.xml, which also means they can differ from one core to another based on the configset being used.
+
+Alternatively "cluster" APIs potentially affect the entire Solr instance or cluster.
+They're registered on the `CoreContainer` object itself.
+It's much less common to provide configuration for these APIs, but it is possible to do so using `solr.xml`.
+
+== V1 APIs
+
+v1 APIs are the primary way that users consume Solr, as our v2 APIs remain "experimental".
+Many new APIs are added as "v2 only", however updates to existing v1 APIs still happen frequently.
+
+v1 APIs exist in Solr as implementations of the `SolrRequestHandler` interface, usually making use of the `RequestHandlerBase` base class.
+RequestHandlers have two methods of primary interest:
+1. `init`, often used to parse per-API configuration or otherwise setup the class
+2. `handleRequest` (or `handleRequestBody` if using `RequestHandlerBase`), the main entrypoint for the API, containing the business logic and constructing the response.
+
+While they do define many aspects of the endpoint's interface (e.g. query parameters, request body format, response format, etc.), RequestHandler's don't actually specify the URL path that they're located at.
+These paths are instead either hardcoded at registration time (see `CoreContainer.load` and `ImplicitPlugins.json`), or specified by users in configuration files (typically `solrconfig.xml`).
+
+== V2 APIs
+
+v2 APIs are currently still "experimental", and not necessarily recommended yet for users.
+But they're approaching parity with v1 and will eventually replace Solr's "RequestHandler"-based APIs.
+
+=== New "JAX-RS" APIs
+
+New v2 APIs in Solr are written in compliance with "JAX-RS", a library and specification that uses annotations to define APIs.
+Many libraries implement the JAX-RS spec: Solr currently uses the implementation provided by the "Jersey" project.
+
+These v2 API definitions consist of two parts: a JAX-RS annotated interface in the `api` module "defining" the API, and a class in `core` "implementing" the interface.
+Separating the API "definition" and "implementation" in this way allows us to only define each API in a single place, and use code generators to produce other API-related bits such as SolrJ code and ref-guide documentation.
+
+=== Writing JAX-RS APIs
+
+Writing a new v2 API may appear daunting, but additions in reality are actually pretty simple:
+
+1. *Create POJO ("Plain Old Java Object") classes as needed to represent the API's request and response*:
+** POJOs are used to represent both the body of the API request (for some `POST` and `PUT` endpoints), as well as the response from the API.
+** Re-use of existing classes here is preferred where possible.  A library of available POJOs can be found in the `org.apache.solr.client.api.model` package of the `api` gradle project.
+** POJO class fields are typically "public" and annotated with the Jackson `@JsonProperty` annotations to allow serialization/deserialization.
+** POJO class fields should also have a Swagger `@Schema` annotation where possible, describing the purpose of the field.  These descriptions are technically non-functional, but add lots of value to our OpenAPI spec and any artifacts generated downstream.
+2. *Find or create an interface to hold the v2 API definition*:
+** API interfaces live in the `org.apache.solr.client.api.endpoint` package of the `api` gradle project.  Interfaces are usually given an "-Api" suffix to indicate their role.
+** If a new API is similar enough to existing APIs, it may make sense to add the new API definition into an existing interface instead of creating a wholly new one.  Use your best judgement.
+3. *Add a method to the chosen interface representing the API*:
+** The method should take an argument representing each path and query parameter (annotated with `@PathParam` or `@QueryParam` as appropriate).  If the API is a `PUT` or `POST` that expects a request body, the method should take the request body POJO as its final argument, annotated with `@RequestBody`.
+** Each method parameter should also be annotated with the Swagger `@Parameter` annotation.  Like the `@Schema` annotation mentioned above, `@Parameter` isn't strictly required for correct operation, but they add lots of value to our OpenAPI spec and generated artifacts.
+** As a return value, the method should return the response-body POJO.
+4. *Futher JAX-RS Annotation*: The interface method in step (3) has specified its inputs and outputs, but several additional annotations are needed to define how users access the API, and to make it play nice with the code-generation done by Solr's build.
+** Each interface must have a `@Path` annotation describing the path that the API is accessed from.  Specific interface methods can also be given `@Path` annotations, making the "effective path" a concatenation of the interface and method-level values.  `@Path` supports a limited regex syntax, and curly-brackets can be used to create named placeholders for path-parameters.
+** Each interface method should be given an HTTP-method annotation (e.g. `@GET`, `@POST`, etc.)
+** Each interface method must be marked with a Swagger `@Operation` annotation.  This annotation is used to provide metadata about the API that appears in the OpenAPI specification and in any artifacts generated from that downstream.  At a minimum, `summary` and `tags` values should be specified on the annotation.  (`tags` is used by our SolrJ code generation to group similar APIs together.  Typically APIs are only given a single tag representing the plural name of the most relevant "resource" (e.g. `tags = {"aliases"}`, `tags = {"replica-properties"}`)
+5. *Create a class implementing the API interface*: Implementation classes live in the `core` gradle project, typically in the `org.apache.solr.handler` package or one of its descendants.
+** Implementing classes must extent `JerseyResource`, and are typically named similarly to the API interface created in (2) above without the "-Api" suffix. e.g. `class AddReplicaProperty extends JerseyResource implements AddReplicaPropertyApi`)
+** Solr's use of Jersey offers us some limited dependency-injection ("DI") capabilities.  Class constructors annotated with `@Inject` can depend on a selection of types made available through DI, such as `CoreContainer`, `SolrQueryRequest`, `SolrCore`, etc.  See the factory-bindings in `JerseyApplications` (or other API classes) for a sense of which types are available for constructor injection.
+** Add a body to your classes method(s).  For the most part this is "normal" Java development.
+6. *Register your API*: APIs must be registered to be available at runtime.  If the v2 API is associated with an existing v1 RequestHandler, the API class name can be added to the handler's `getJerseyResources` method.  If there is no associated RequestHandler, the API should be registered similar to other APIs in `CoreContainer.load`.
+
+A good example for each of these steps can be seen in Solr's v2 "add-replica-property" API, which has a defining interface https://github.com/apache/solr/blob/9426902acb7081a2e9a1fa29699c5286459e1365/solr/api/src/java/org/apache/solr/client/api/endpoint/AddReplicaPropertyApi.java[AddReplicaPropertyApi], an implementing class https://github.com/apache/solr/blob/9426902acb7081a2e9a1fa29699c5286459e1365/solr/core/src/java/org/apache/solr/handler/admin/api/AddReplicaProperty.java[AddReplicaProperty], and the two POJOs https://github.com/apache/solr/blob/main/solr/api/src/java/org/apache/solr/client/api/model/AddReplicaPropertyRequestBody.java[AddReplicaPropertyRequestBody] and https://github.com/apache/solr/blob/main/solr/api/src/java/org/apache/solr/client/api/model/SolrJerseyResponse.java[SolrJerseyResponse].
+
+=== Legacy v2 API Framework
+
+While we've settled on JAX-RS as our framework for defining v2 APIs going forward, Solr still retains many v2 APIs that were written using an older homegrown framework.
+This framework defines APIs using annotations (e.g. `@EndPoint`) similar to those used by JAX-RS, but lacks the full range of features and 3rd-party tooling.
+We're in the process of migrating these API definitions to JAX-RS and hope to remove all support for this legacy framework in a future release.
+

dev-docs/cloud-script.adoc

@@ -18,7 +18,7 @@ Typical usage of the script is:
 1. Copy the script to a location outside the `./solr/` working copy such as `<GIT_CHECKOUT>/../solr-testing`. Trying to use it from within the working copy will leave you fighting version control to avoid checking in files.
 2. Edit the script to provide a proper back reference to your working copy for `DEFAULT_VCS_WORKSPACE` (the default is `../code/solr`)
 3. Start a local zookeeper instance. This zookeeper must be running on localhost, but the port can be adjusted with a `-z` option passed to the script.  `docker run -p 2181:2181 zookeeper` is one option to start up a local Zookeeper.
-4. run ./cloud.sh new -r
+4. run `./cloud.sh new -r`
 
 The command in the final step will:
 

dev-tools/doap/solr.rdf

@@ -68,6 +68,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+         <Version>
+           <name>solr-9.4.0</name>
+           <created>2023-10-14</created>
+           <revision>9.4.0</revision>
+         </Version>
+    </release>
     <release>
          <Version>
            <name>solr-9.3.0</name>

dev-tools/scripts/addVersion.py

@@ -44,7 +44,7 @@ def edit(buffer, match, line):
   print('done' if changed else 'uptodate')
 
 def update_solrversion_class(new_version):
-  filename = 'solr/core/src/java/org/apache/solr/util/SolrVersion.java'
+  filename = 'solr/api/src/java/org/apache/solr/client/api/util/SolrVersion.java'
   print('  changing version to %s...' % new_version.dot, end='', flush=True)
   constant_prefix = 'public static final String LATEST_STRING = "(.*?)"'
   matcher = re.compile(constant_prefix)
@@ -104,7 +104,7 @@ def edit(buffer, match, line):
 
 def check_solr_version_class_tests():
   print('  checking solr version tests...', end='', flush=True)
-  run('./gradlew -p solr/core test --tests TestSolrVersion')
+  run('./gradlew -p solr/api test --tests TestSolrVersion')
   print('ok')
 
 def check_lucene_match_version_tests():

dev-tools/scripts/buildAndPushRelease.py

@@ -128,7 +128,7 @@ def prepare(root, version, pause_before_sign, mf_username, gpg_key_id, gpg_passw
   if dev_mode:
     cmd += ' -Pvalidation.git.failOnModified=false'
   if gpg_key_id is None:
-    cmd += ' -Psign=false -x signJarsPublication'  # Disable signing if no key provided to script
+    cmd += ' -Psign=false'  # Disable signing if no key provided to script
   else:
     cmd += ' -Psign --max-workers 2'
     if sign_gradle:

dev-tools/scripts/releaseWizard.yaml

@@ -205,7 +205,7 @@ groups:
   title: Prerequisites
   description: |
     Releasing software requires thorough understanding of the process and careful execution,
-    as it is easy to make mistakes. It also requires an environtment and tools such as gpg
+    as it is easy to make mistakes. It also requires an environment and tools such as gpg
     correctly setup. This section makes sure you're in good shape for the job!
   todos:
   - !Todo
@@ -526,7 +526,7 @@ groups:
     - major
     - minor
     links:
-    - https://cwiki.apache.org/confluence/display/LUCENEJAVA/JenkinsReleaseBuilds
+    - https://cwiki.apache.org/confluence/display/SOLR/JenkinsReleaseBuilds+-+Solr
   - !Todo
     id: inform_devs
     title: Inform Devs of the new Release Branch
@@ -544,7 +544,7 @@ groups:
 
       NOTICE:
 
-      Branch {{ release_branch }} has been cut and versions updated to {{ release_version_major }}.{{ release_version_minor + 1 }} on stable branch.
+      Branch {{ release_branch }} has been cut and versions updated to {{ release_version_major }}.{{ release_version_minor + 1 }} on the stable branch.
 
       Please observe the normal rules:
 
@@ -636,7 +636,7 @@ groups:
       Ask on dev@ for input. Ideally the timing of this request mostly coincides with the
       release branch creation. It's a good idea to remind the devs of this later in the release too.
 
-      NOTE: Do not add every single JIRA issue, but distill the Release note into important changes!
+      NOTE: Do not add every single JIRA issue, but distill the release notes into important changes!
     links:
     - https://cwiki.apache.org/confluence/display/SOLR/Release+Notes
   - !Todo
@@ -660,7 +660,7 @@ groups:
     description: |-
       To make sure a new release is not announced before the reference guide is complete, add a new
       Blocker JIRA issue for version {{ release_version }} with subject
-      "Complete Major changes and Upgrade Notes in RefGudie for {{ release_version }}".
+      "Complete Major changes and Upgrade Notes in RefGuide for {{ release_version }}".
       Note that this does not block the RC and voting process, but will block announcing the release.
     types:
     - major
@@ -1588,6 +1588,7 @@ groups:
       Fill in the same date that you used for the release in previous steps.
       Do not use a product name prefix for the version, as this is the main release of the Solr PMC.
       Just use the version of this release: {{ release_version }}
+      Note: you have to be a member of the Solr PMC to be able to complete this step.
     links:
       - https://reporter.apache.org/addrelease.html?solr
 - !TodoGroup

dev-tools/scripts/smokeTestRelease.py

@@ -606,7 +606,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
     expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
     expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'versions.lock', 'versions.props']
     expected_src_solr_files = ['build.gradle']
-    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
+    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
     is_in_list(in_root_folder, expected_src_root_folders)
     is_in_list(in_root_folder, expected_src_root_files)
     is_in_list(in_solr_folder, expected_src_solr_folders)

gradle/hacks/turbocharge-jvm-opts.gradle

@@ -52,8 +52,8 @@ allprojects {
                 // are not part of up-to-date checks but these are internal JVM flags so we
                 // don't care.
                 //
-                // Pass VM options via -J only with a custom javaHome AND when java toolchains are not used.
-                if (task.options.forkOptions.javaHome != null && task.javaCompiler.getOrNull() == null) {
+                // Pass VM options via -J when a custom javaHome is used and we're in fork mode.
+                if (task.options.fork && task.options.forkOptions.javaHome != null) {
                     return vmOpts.collect {"-J" + it}
                 } else {
                     return vmOpts

gradle/maven/defaults-maven.gradle

@@ -24,6 +24,7 @@
 configure(rootProject) {
   ext {
     published = [
+        ":solr:api",
         ":solr:core",
         ":solr:solrj",
         ":solr:solrj-streaming",
@@ -176,7 +177,7 @@ configure(subprojects.findAll { it.path in rootProject.published }) { prj ->
         if (project(":solr:distribution").ext.useGpgForSigning) {
           useGpgCmd()
         }
-        required { !version.endsWith("SNAPSHOT") }
+        required { project(":solr:distribution").ext.withSignedArtifacts }
         sign publishing.publications.jars
       }
     }