From 844cde9a4f088f00d108b9c74be967aaf03bdff0 Mon Sep 17 00:00:00 2001 From: patsonluk Date: Mon, 17 Jun 2024 13:43:01 -0700 Subject: [PATCH 001/172] SOLR-17269: Do not publish synthetic solr core (of Coordinator node) to ZK (#2438) Removed logic that registers the synthetic collection/core on a Coordinator node to Zookeeper. This simplifies the code flow and avoids confusion to external tools that mistakenly recognize the synthetic collection as an actual collection. This is achieved by the introduction of SyntheticSolrCore which is created by CoordinatorHttpSolrCall. SyntheticSolrCore provides a shortcut creation route that bypasses ZK registration. --- solr/CHANGES.txt | 2 + .../apache/solr/cloud/ZkConfigSetService.java | 39 +-- .../java/org/apache/solr/core/SolrCore.java | 2 +- .../apache/solr/core/SyntheticSolrCore.java | 75 ++++++ .../solr/servlet/CoordinatorHttpSolrCall.java | 246 +++++------------- .../solr/search/TestCoordinatorRole.java | 42 +-- 6 files changed, 164 insertions(+), 242 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index be2cba6139f..bd113b9974c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -136,6 +136,8 @@ Optimizations * SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac) +* SOLR-17269: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (Patson Luk) + Bug Fixes --------------------- * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java index 138d729d9ee..5057e62613f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java @@ -26,12 +26,9 @@ import java.util.Map; import java.util.Objects; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.cloud.api.collections.CreateCollectionCmd; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetProperties; @@ -71,36 +68,18 @@ public ZkConfigSetService(SolrZkClient zkClient) { @Override public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) { - final String colName = cd.getCollectionName(); - - // For back compat with cores that can create collections without the collections API - try { - if (!zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + colName, true)) { - // TODO remove this functionality or maybe move to a CLI mechanism - log.warn( - "Auto-creating collection (in ZK) from core descriptor (on disk). This feature may go away!"); - CreateCollectionCmd.createCollectionZkNode( - zkController.getSolrCloudManager().getDistribStateManager(), - colName, - cd.getCloudDescriptor().getParams(), - zkController.getCoreContainer().getConfigSetService()); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "Interrupted auto-creating collection", e); - } catch (KeeperException e) { - throw new ZooKeeperException( - SolrException.ErrorCode.SERVER_ERROR, "Failure auto-creating collection", e); + // Currently, cd.getConfigSet() is always null. Except that it's explicitly set by + // {@link org.apache.solr.core.SyntheticSolrCore}. + // Should we consider setting it for all cores as a part of CoreDescriptor creation/loading + // process? + if (cd.getConfigSet() == null) { + String configSetName = + zkController.getClusterState().getCollection(cd.getCollectionName()).getConfigName(); + cd.setConfigSet(configSetName); } - // The configSet is read from ZK and populated. Ignore CD's pre-existing configSet; only - // populated in standalone - String configSetName = zkController.getClusterState().getCollection(colName).getConfigName(); - cd.setConfigSet(configSetName); - return new ZkSolrResourceLoader( - cd.getInstanceDir(), configSetName, parentLoader.getClassLoader(), zkController); + cd.getInstanceDir(), cd.getConfigSet(), parentLoader.getClassLoader(), zkController); } @Override diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 767fb34d316..e4bee2e8502 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1230,7 +1230,7 @@ private SolrCore( } /** Set UpdateLog to buffer updates if the slice is in construction. */ - private void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) { + protected void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) { if (coreContainer != null && coreContainer.isZooKeeperAware()) { if (reqHandlers.get("/get") == null) { diff --git a/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java b/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java new file mode 100644 index 00000000000..e97c5780d54 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.core; + +import java.nio.file.Path; +import java.util.Map; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.rest.RestManager; + +/** + * A synthetic core that is created only in memory and not registered against Zookeeper. + * + *

This is only used in Coordinator node to support a subset of SolrCore functionalities required + * by Coordinator operations such as aggregating and writing out response and providing configset + * info. + * + *

There should only be one instance of SyntheticSolrCore per configset + */ +public class SyntheticSolrCore extends SolrCore { + public SyntheticSolrCore(CoreContainer coreContainer, CoreDescriptor cd, ConfigSet configSet) { + super(coreContainer, cd, configSet); + } + + public static SyntheticSolrCore createAndRegisterCore( + CoreContainer coreContainer, String syntheticCoreName, String configSetName) { + Map coreProps = Map.of(CoreAdminParams.COLLECTION, syntheticCoreName); + + CoreDescriptor syntheticCoreDescriptor = + new CoreDescriptor( + syntheticCoreName, + Path.of(coreContainer.getSolrHome(), syntheticCoreName), + coreProps, + coreContainer.getContainerProperties(), + coreContainer.getZkController()); + syntheticCoreDescriptor.setConfigSet(configSetName); + ConfigSet coreConfig = + coreContainer.getConfigSetService().loadConfigSet(syntheticCoreDescriptor); + syntheticCoreDescriptor.setConfigSetTrusted(coreConfig.isTrusted()); + SyntheticSolrCore syntheticCore = + new SyntheticSolrCore(coreContainer, syntheticCoreDescriptor, coreConfig); + coreContainer.registerCore(syntheticCoreDescriptor, syntheticCore, false, false); + + return syntheticCore; + } + + @Override + protected void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) { + // no updates to SyntheticSolrCore + } + + @Override + protected RestManager initRestManager() throws SolrException { + // returns an initialized RestManager. As init routines requires reading configname of the + // core's collection from ZK + // which synthetic core is not registered in ZK. + // We do not expect RestManager ops on Coordinator Nodes + return new RestManager(); + } +} diff --git a/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java index ec00107b717..da8ca523fce 100644 --- a/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java @@ -18,32 +18,25 @@ package org.apache.solr.servlet; import java.lang.invoke.MethodHandles; -import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.solr.api.CoordinatorV2HttpSolrCall; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.api.collections.Assign; -import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SyntheticSolrCore; import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.request.DelegatingSolrQueryRequest; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,198 +81,70 @@ public static SolrCore getCore( String syntheticCoreName = factory.collectionVsCoreNameMapping.get(collectionName); if (syntheticCoreName != null) { SolrCore syntheticCore = solrCall.cores.getCore(syntheticCoreName); - setMDCLoggingContext(collectionName); + setMdcLoggingContext(collectionName); return syntheticCore; } else { + // first time loading this collection ZkStateReader zkStateReader = solrCall.cores.getZkController().getZkStateReader(); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollectionOrNull(collectionName, true); - SolrCore core = null; - if (coll != null) { - String confName = coll.getConfigName(); - String syntheticCollectionName = getSyntheticCollectionName(confName); - DocCollection syntheticColl = clusterState.getCollectionOrNull(syntheticCollectionName); - synchronized (CoordinatorHttpSolrCall.class) { - if (syntheticColl == null) { - // no synthetic collection for this config, let's create one - if (log.isInfoEnabled()) { - log.info( - "synthetic collection: {} does not exist, creating.. ", syntheticCollectionName); - } - - SolrException createException = null; - try { - createColl(syntheticCollectionName, solrCall.cores, confName); - } catch (SolrException exception) { - // concurrent requests could have created the collection hence causing collection - // exists - // exception - createException = exception; - } finally { - syntheticColl = - zkStateReader.getClusterState().getCollectionOrNull(syntheticCollectionName); - } - - // then indeed the collection was not created properly, either by this or other - // concurrent - // requests - if (syntheticColl == null) { - if (createException != null) { - throw createException; // rethrow the exception since such collection was not - // created - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Could not locate synthetic collection [" - + syntheticCollectionName - + "] after creation!"); - } - } - } - - // get docCollection again to ensure we get the fresh state - syntheticColl = - zkStateReader.getClusterState().getCollectionOrNull(syntheticCollectionName); - List nodeNameSyntheticReplicas = - syntheticColl.getReplicas(solrCall.cores.getZkController().getNodeName()); - if (nodeNameSyntheticReplicas == null || nodeNameSyntheticReplicas.isEmpty()) { - // this node does not have a replica. add one - if (log.isInfoEnabled()) { - log.info( - "this node does not have a replica of the synthetic collection: {} , adding replica ", - syntheticCollectionName); - } + if (coll == null) { // querying on a non-existent collection, it could have been removed + log.info( + "Cannot find collection {} to proxy call to, it could have been deleted", + collectionName); + return null; + } - addReplica(syntheticCollectionName, solrCall.cores); - } + String confName = coll.getConfigName(); + syntheticCoreName = getSyntheticCoreNameFromConfig(confName); + + SolrCore syntheticCore; + synchronized (CoordinatorHttpSolrCall.class) { + CoreContainer coreContainer = solrCall.cores; + syntheticCore = coreContainer.getCore(syntheticCoreName); + if (syntheticCore == null) { + // first time loading this config set + log.info("Loading synthetic core for config set {}", confName); + syntheticCore = + SyntheticSolrCore.createAndRegisterCore( + coreContainer, syntheticCoreName, coll.getConfigName()); + // getting the core should open it + syntheticCore.open(); + } - // still have to ensure that it's active, otherwise super.getCoreByCollection - // will return null and then CoordinatorHttpSolrCall will call getCore again - // hence creating a calling loop - try { - zkStateReader.waitForState( - syntheticCollectionName, - 10, - TimeUnit.SECONDS, - docCollection -> { - List replicas = - docCollection.getReplicas(solrCall.cores.getZkController().getNodeName()); - if (replicas == null || replicas.isEmpty()) { + factory.collectionVsCoreNameMapping.put(collectionName, syntheticCore.getName()); + + // for the watcher, only remove on collection deletion (ie collection == null), since + // watch from coordinator is collection specific + coreContainer + .getZkController() + .getZkStateReader() + .registerDocCollectionWatcher( + collectionName, + collection -> { + if (collection == null) { + factory.collectionVsCoreNameMapping.remove(collectionName); + return true; + } else { return false; } - for (Replica nodeNameSyntheticReplica : replicas) { - if (nodeNameSyntheticReplica.getState() == Replica.State.ACTIVE) { - return true; - } - } - return false; }); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to wait for active replica for synthetic collection [" - + syntheticCollectionName - + "]", - e); - } - } - - core = solrCall.getCoreByCollection(syntheticCollectionName, isPreferLeader); - if (core != null) { - factory.collectionVsCoreNameMapping.put(collectionName, core.getName()); - // for the watcher, only remove on collection deletion (ie collection == null), since - // watch from coordinator is collection specific - solrCall - .cores - .getZkController() - .getZkStateReader() - .registerDocCollectionWatcher( - collectionName, - collection -> { - if (collection == null) { - factory.collectionVsCoreNameMapping.remove(collectionName); - return true; - } else { - return false; - } - }); - if (log.isDebugEnabled()) { - log.debug("coordinator node, returns synthetic core: {}", core.getName()); - } - } - setMDCLoggingContext(collectionName); - return core; } - return null; + setMdcLoggingContext(collectionName); + if (log.isDebugEnabled()) { + log.debug("coordinator node, returns synthetic core: {}", syntheticCore.getName()); + } + return syntheticCore; } } - public static String getSyntheticCollectionName(String configName) { + public static String getSyntheticCollectionNameFromConfig(String configName) { return SYNTHETIC_COLL_PREFIX + configName; } - /** - * Overrides the MDC context as the core set was synthetic core, which does not reflect the - * collection being operated on - */ - private static void setMDCLoggingContext(String collectionName) { - MDCLoggingContext.setCollection(collectionName); - - // below is irrelevant for call to coordinator - MDCLoggingContext.setCoreName(null); - MDCLoggingContext.setShard(null); - MDCLoggingContext.setCoreName(null); - } - - private static void addReplica(String syntheticCollectionName, CoreContainer cores) { - SolrQueryResponse rsp = new SolrQueryResponse(); - try { - CollectionAdminRequest.AddReplica addReplicaRequest = - CollectionAdminRequest.addReplicaToShard(syntheticCollectionName, "shard1") - // we are fixing the name, so that no two replicas are created in the same node - .setNode(cores.getZkController().getNodeName()); - addReplicaRequest.setWaitForFinalState(true); - cores - .getCollectionsHandler() - .handleRequestBody(new LocalSolrQueryRequest(null, addReplicaRequest.getParams()), rsp); - if (rsp.getValues().get("success") == null) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Could not auto-create collection: " + Utils.toJSONString(rsp.getValues())); - } - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - } - - private static void createColl( - String syntheticCollectionName, CoreContainer cores, String confName) { - SolrQueryResponse rsp = new SolrQueryResponse(); - try { - CollectionAdminRequest.Create collCreationRequest = - CollectionAdminRequest.createCollection(syntheticCollectionName, confName, 1, 1) - .setCreateNodeSet(cores.getZkController().getNodeName()); - collCreationRequest.setWaitForFinalState(true); - SolrParams params = collCreationRequest.getParams(); - if (log.isInfoEnabled()) { - log.info("sending collection admin command : {}", Utils.toJSONString(params)); - } - cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null, params), rsp); - if (rsp.getValues().get("success") == null) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Could not create :" - + syntheticCollectionName - + " collection: " - + Utils.toJSONString(rsp.getValues())); - } - } catch (SolrException e) { - throw e; - - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } + public static String getSyntheticCoreNameFromConfig(String configName) { + return getSyntheticCollectionNameFromConfig(configName) + "_core"; } @Override @@ -298,7 +163,9 @@ protected String getCoreOrColName() { public static SolrQueryRequest wrappedReq( SolrQueryRequest delegate, String collectionName, HttpSolrCall httpSolrCall) { Properties p = new Properties(); - p.put(CoreDescriptor.CORE_COLLECTION, collectionName); + if (collectionName != null) { + p.put(CoreDescriptor.CORE_COLLECTION, collectionName); + } p.put(CloudDescriptor.REPLICA_TYPE, Replica.Type.PULL.toString()); p.put(CoreDescriptor.CORE_SHARD, "_"); @@ -318,6 +185,19 @@ public CloudDescriptor getCloudDescriptor() { }; } + /** + * Overrides the MDC context as the core set was synthetic core, which does not reflect the + * collection being operated on + */ + private static void setMdcLoggingContext(String collectionName) { + MDCLoggingContext.setCollection(collectionName); + + // below is irrelevant for call to coordinator + MDCLoggingContext.setCoreName(null); + MDCLoggingContext.setShard(null); + MDCLoggingContext.setCoreName(null); + } + // The factory that creates an instance of HttpSolrCall public static class Factory implements SolrDispatchFilter.HttpSolrCallFactory { private final Map collectionVsCoreNameMapping = new ConcurrentHashMap<>(); diff --git a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java index f55aca49ae8..70835d7e884 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java +++ b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java @@ -24,11 +24,9 @@ import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Random; -import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -62,6 +60,8 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.core.NodeRoles; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SyntheticSolrCore; import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.servlet.CoordinatorHttpSolrCall; import org.slf4j.Logger; @@ -76,7 +76,6 @@ public void testSimple() throws Exception { try { CloudSolrClient client = cluster.getSolrClient(); String COLLECTION_NAME = "test_coll"; - String SYNTHETIC_COLLECTION = CoordinatorHttpSolrCall.getSyntheticCollectionName("conf"); CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf", 2, 2) .process(cluster.getSolrClient()); cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); @@ -105,14 +104,20 @@ public void testSimple() throws Exception { assertEquals(10, rslt.getResults().size()); + String SYNTHETIC_COLLECTION = + CoordinatorHttpSolrCall.getSyntheticCollectionNameFromConfig("conf"); DocCollection collection = cluster.getSolrClient().getClusterStateProvider().getCollection(SYNTHETIC_COLLECTION); - assertNotNull(collection); - - Set expectedNodes = new HashSet<>(); - expectedNodes.add(coordinatorJetty.getNodeName()); - collection.forEachReplica((s, replica) -> expectedNodes.remove(replica.getNodeName())); - assertTrue(expectedNodes.isEmpty()); + // this should be empty as synthetic collection does not register with ZK + assertNull(collection); + + String syntheticCoreName = CoordinatorHttpSolrCall.getSyntheticCoreNameFromConfig("conf"); + try (SolrCore syntheticCore = + coordinatorJetty.getCoreContainer().getCore(syntheticCoreName)) { + assertNotNull(syntheticCore); + assertTrue(syntheticCore instanceof SyntheticSolrCore); + assertEquals("conf", syntheticCore.getCoreDescriptor().getConfigSet()); + } } finally { cluster.shutdown(); } @@ -124,7 +129,6 @@ public void testMultiCollectionMultiNode() throws Exception { try { CloudSolrClient client = cluster.getSolrClient(); String COLLECTION_NAME = "test_coll"; - String SYNTHETIC_COLLECTION = CoordinatorHttpSolrCall.getSyntheticCollectionName("conf"); for (int j = 1; j <= 10; j++) { String collname = COLLECTION_NAME + "_" + j; CollectionAdminRequest.createCollection(collname, "conf", 2, 2) @@ -170,15 +174,6 @@ public void testMultiCollectionMultiNode() throws Exception { assertEquals(10, rslt.getResults().size()); } - - DocCollection collection = - cluster.getSolrClient().getClusterStateProvider().getCollection(SYNTHETIC_COLLECTION); - assertNotNull(collection); - - int coordNode1NumCores = coordinatorJetty1.getCoreContainer().getNumAllCores(); - assertEquals("Unexpected number of cores found for coordinator node", 1, coordNode1NumCores); - int coordNode2NumCores = coordinatorJetty2.getCoreContainer().getNumAllCores(); - assertEquals("Unexpected number of cores found for coordinator node", 1, coordNode2NumCores); } finally { cluster.shutdown(); } @@ -573,15 +568,6 @@ public void testConcurrentAccess() throws Exception { testFuture.get(); // check for any exceptions/failures } - // number of replicas created in the synthetic collection should be one per coordinator node - assertEquals( - COORDINATOR_NODE_COUNT, - client - .getClusterState() - .getCollection(CoordinatorHttpSolrCall.getSyntheticCollectionName("conf")) - .getReplicas() - .size()); - executorService.shutdown(); executorService.awaitTermination(10, TimeUnit.SECONDS); } finally { From fc0d84afaa8b49bd0515f796abd901e5150d5982 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 18 Jun 2024 14:26:05 -0400 Subject: [PATCH 002/172] SOLR-17331: More optimal placements with OrderedNodePlacementPlugin (#2515) - Move tests, adding tests for the simple plugin --- solr/CHANGES.txt | 2 + .../plugins/OrderedNodePlacementPlugin.java | 87 ++-- .../solr/cloud/MigrateReplicasTest.java | 8 +- .../MinimizeCoresPlacementFactoryTest.java | 2 +- .../plugins/SimplePlacementFactoryTest.java | 370 ++++++++++++++++++ 5 files changed, 426 insertions(+), 43 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cluster/placement/plugins/SimplePlacementFactoryTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index bd113b9974c..03876f1a414 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -129,6 +129,8 @@ Improvements * SOLR-17109: Give security manager explicit read access to sharedLib (Tomás Fernández Löbbe via Eric Pugh) +* SOLR-17331: OrderedNodePlacementPlugin will give an even more optimal replica placements during ReplicaMigration commands (Houston Putman, Yohann Callea) + Optimizations --------------------- * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java b/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java index 5c8d195954e..86ca79526b3 100644 --- a/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java +++ b/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java @@ -69,23 +69,26 @@ public List computePlacements( Set allNodes = new HashSet<>(); Set allCollections = new HashSet<>(); - Deque pendingRequests = new ArrayDeque<>(requests.size()); + // This helps us keep track if we have made a full lap of the outstanding requests + // without doing a placement or not. + int placementCount = 0; + Deque outstandingRequests = new ArrayDeque<>(requests.size()); for (PlacementRequest request : requests) { - PendingPlacementRequest pending = new PendingPlacementRequest(request); - pendingRequests.add(pending); + OutstandingPlacementRequest outstanding = new OutstandingPlacementRequest(request); + outstandingRequests.add(outstanding); placementPlans.add( placementContext .getPlacementPlanFactory() - .createPlacementPlan(request, pending.getComputedPlacementSet())); + .createPlacementPlan(request, outstanding.getComputedPlacementSet())); allNodes.addAll(request.getTargetNodes()); allCollections.add(request.getCollection()); } Collection weightedNodes = getWeightedNodes(placementContext, allNodes, allCollections, true).values(); - while (!pendingRequests.isEmpty()) { - PendingPlacementRequest request = pendingRequests.poll(); - if (!request.isPending()) { + while (!outstandingRequests.isEmpty()) { + OutstandingPlacementRequest request = outstandingRequests.poll(); + if (request.isComplete()) { continue; } @@ -94,9 +97,9 @@ public List computePlacements( SolrCollection solrCollection = request.getCollection(); // Now place all replicas of all shards on available nodes - for (String shardName : request.getPendingShards()) { - for (Replica.ReplicaType replicaType : request.getPendingReplicaTypes(shardName)) { - int replicaCount = request.getPendingReplicas(shardName, replicaType); + for (String shardName : request.getOutstandingShards()) { + for (Replica.ReplicaType replicaType : request.getOutstandingReplicaTypes(shardName)) { + int replicaCount = request.getOutstandingReplicas(shardName, replicaType); if (log.isDebugEnabled()) { log.debug( "Placing {} replicas for Collection: {}, Shard: {}, ReplicaType: {}", @@ -131,12 +134,13 @@ public List computePlacements( // options than we have replicas to place, that's ok, because the replicas will likely // be put on all the tie options. // - // Only skip the request if it can be requeued, and there are other pending requests to - // compute. + // Only skip the request if it can be requeued, and there are other outstanding requests + // to + // compute. If the next outstanding request cannot be requeued, int numWeightTies = nodesForReplicaType.peekTies(); - if (!pendingRequests.isEmpty() - && request.canBeRequeued() - && numWeightTies > (replicaCount - replicasPlaced)) { + if (numWeightTies > (replicaCount - replicasPlaced) + && !outstandingRequests.isEmpty() + && request.canBeRequeued(placementCount)) { log.debug( "There is a tie for best weight. There are more options ({}) than replicas to place ({}), so try this placement request later: {}", numWeightTies, @@ -151,6 +155,7 @@ public List computePlacements( node.addReplica( createProjectedReplica(solrCollection, shardName, replicaType, node.getNode())); replicasPlaced += 1; + placementCount += 1; request.addPlacement( placementContext .getPlacementPlanFactory() @@ -185,9 +190,9 @@ public List computePlacements( } } } - if (request.isPending()) { - request.requeue(); - pendingRequests.add(request); + if (!request.isComplete()) { + request.requeue(placementCount); + outstandingRequests.add(request); } } return placementPlans; @@ -808,8 +813,8 @@ public void resortAll() { } /** Context for a placement request still has replicas that need to be placed. */ - static class PendingPlacementRequest { - boolean hasBeenRequeued; + static class OutstandingPlacementRequest { + int requeuedAtPlacementCount; final SolrCollection collection; @@ -821,8 +826,8 @@ static class PendingPlacementRequest { // A live view on how many replicas still need to be placed for each shard & replica type final Map> replicasToPlaceForShards; - public PendingPlacementRequest(PlacementRequest request) { - hasBeenRequeued = false; + public OutstandingPlacementRequest(PlacementRequest request) { + requeuedAtPlacementCount = -1; collection = request.getCollection(); targetNodes = request.getTargetNodes(); Set shards = request.getShardNames(); @@ -843,13 +848,13 @@ public PendingPlacementRequest(PlacementRequest request) { } /** - * Determine if this request is not yet complete, and there are requested replicas that have not + * Determine if this request is complete, and there are no requested replicas that have not yet * had placements computed. * - * @return if there are still replica placements that need to be computed + * @return if all replica placements have been computed */ - public boolean isPending() { - return !replicasToPlaceForShards.isEmpty(); + public boolean isComplete() { + return replicasToPlaceForShards.isEmpty(); } public SolrCollection getCollection() { @@ -864,7 +869,7 @@ public boolean isTargetingNode(WeightedNode node) { * The set of ReplicaPlacements computed for this request. * *

The list that is returned is the same list used internally, so it will be augmented until - * {@link #isPending()} returns false. + * {@link #isComplete()} returns true. * * @return The live set of replicaPlacements for this request. */ @@ -879,7 +884,7 @@ public Set getComputedPlacementSet() { * * @return list of unfinished shards */ - public Collection getPendingShards() { + public Collection getOutstandingShards() { return new ArrayList<>(replicasToPlaceForShards.keySet()); } @@ -890,7 +895,7 @@ public Collection getPendingShards() { * @param shard name of the shard to check for uncomputed placements * @return the set of unfinished replica types */ - public Collection getPendingReplicaTypes(String shard) { + public Collection getOutstandingReplicaTypes(String shard) { return Optional.ofNullable(replicasToPlaceForShards.get(shard)) .map(Map::keySet) // Use a sorted TreeSet to make sure that tests are repeatable @@ -906,30 +911,36 @@ public Collection getPendingReplicaTypes(String shard) { * @param type type of replica to be placed * @return the number of replicas that have not yet had placements computed */ - public int getPendingReplicas(String shard, Replica.ReplicaType type) { + public int getOutstandingReplicas(String shard, Replica.ReplicaType type) { return Optional.ofNullable(replicasToPlaceForShards.get(shard)) .map(m -> m.get(type)) .orElse(0); } /** - * Currently, only of requeue is allowed per pending request. + * The request can be requeued as long as there have been placements since the last time it was + * requeued. If no placements have been made since, we will not allow a requeue which will force + * placements to be determined. This removes the possibility of a never-ending loop. * * @return true if the request has not been requeued already */ - public boolean canBeRequeued() { - return !hasBeenRequeued; + public boolean canBeRequeued(int placementCount) { + return requeuedAtPlacementCount < placementCount; } - /** Let the pending request know that it has been requeued */ - public void requeue() { - hasBeenRequeued = true; + /** + * Let the request know that it has been requeued + * + * @param placementCount the number of placements made at the time of the requeue + */ + public void requeue(int placementCount) { + requeuedAtPlacementCount = placementCount; } /** - * Track the given replica placement for this pending request. + * Track the given replica placement for this request. * - * @param replica placement that has been made for the pending request + * @param replica placement that has been made for the request */ public void addPlacement(ReplicaPlacement replica) { computedPlacements.add(replica); diff --git a/solr/core/src/test/org/apache/solr/cloud/MigrateReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/MigrateReplicasTest.java index f96b621c023..39ccf43fb53 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MigrateReplicasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MigrateReplicasTest.java @@ -237,7 +237,7 @@ public void test() throws Exception { } @Test - public void testGoodSpreadDuringAssignWithNoTarget() throws Exception { + public void testWithNoTarget() throws Exception { configureCluster(5) .addConfig( "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) @@ -300,9 +300,9 @@ public void testGoodSpreadDuringAssignWithNoTarget() throws Exception { } for (String node : eventualTargetNodes) { - assertEquals( - "The non-source node '" + node + "' has the wrong number of replicas after the migration", - 2, + assertNotEquals( + "The non-source node '" + node + "' should not receive all replicas from the migration", + 4, collection.getReplicas(node).size()); } } diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/plugins/MinimizeCoresPlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/plugins/MinimizeCoresPlacementFactoryTest.java index d85d30c8a13..bb058a734bb 100644 --- a/solr/core/src/test/org/apache/solr/cluster/placement/plugins/MinimizeCoresPlacementFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/cluster/placement/plugins/MinimizeCoresPlacementFactoryTest.java @@ -39,7 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Unit test for {@link AffinityPlacementFactory} */ +/** Unit test for {@link MinimizeCoresPlacementFactoryTest} */ public class MinimizeCoresPlacementFactoryTest extends AbstractPlacementFactoryTest { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/plugins/SimplePlacementFactoryTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/plugins/SimplePlacementFactoryTest.java new file mode 100644 index 00000000000..2f4d81f8658 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cluster/placement/plugins/SimplePlacementFactoryTest.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cluster.placement.plugins; + +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.solr.cluster.Node; +import org.apache.solr.cluster.Shard; +import org.apache.solr.cluster.SolrCollection; +import org.apache.solr.cluster.placement.BalancePlan; +import org.apache.solr.cluster.placement.Builders; +import org.apache.solr.cluster.placement.PlacementContext; +import org.apache.solr.cluster.placement.PlacementPlan; +import org.apache.solr.cluster.placement.PlacementPlugin; +import org.apache.solr.cluster.placement.PlacementRequest; +import org.apache.solr.cluster.placement.ReplicaPlacement; +import org.apache.solr.cluster.placement.impl.BalanceRequestImpl; +import org.apache.solr.cluster.placement.impl.PlacementRequestImpl; +import org.apache.solr.common.cloud.ReplicaCount; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Unit test for {@link SimplePlacementFactoryTest} */ +public class SimplePlacementFactoryTest extends AbstractPlacementFactoryTest { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private PlacementPlugin plugin; + + @Before + public void setupPlugin() { + configurePlugin(); + } + + private void configurePlugin() { + SimplePlacementFactory factory = new SimplePlacementFactory(); + plugin = factory.createPluginInstance(); + } + + @Test + public void testBasicPlacementNewCollection() throws Exception { + testBasicPlacementInternal(false); + } + + @Test + public void testBasicPlacementExistingCollection() throws Exception { + testBasicPlacementInternal(true); + } + + /** + * When this test places a replica for a new collection, it should pick the node with fewer cores. + * + *

+ * + *

When it places a replica for an existing collection, it should pick the node with fewer + * cores that doesn't already have a replica for the shard. + */ + private void testBasicPlacementInternal(boolean hasExistingCollection) throws Exception { + String collectionName = "basicCollection"; + + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(2); + List nodeBuilders = clusterBuilder.getLiveNodeBuilders(); + + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName); + + if (hasExistingCollection) { + // Existing collection has replicas for its shards and is visible in the cluster state + collectionBuilder.initializeShardsReplicas(1, 1, 0, 0, nodeBuilders); + clusterBuilder.addCollection(collectionBuilder); + } else { + // New collection to create has the shards defined but no replicas and is not present in + // cluster state + collectionBuilder.initializeShardsReplicas(1, 0, 0, 0, List.of()); + } + + PlacementContext placementContext = clusterBuilder.buildPlacementContext(); + + SolrCollection solrCollection = collectionBuilder.build(); + List liveNodes = clusterBuilder.buildLiveNodes(); + + // Place a new replica for the (only) existing shard of the collection + PlacementRequestImpl placementRequest = + new PlacementRequestImpl( + solrCollection, + Set.of(solrCollection.shards().iterator().next().getShardName()), + new HashSet<>(liveNodes), + ReplicaCount.of(1, 0, 0)); + + PlacementPlan pp = plugin.computePlacement(placementRequest, placementContext); + + assertEquals(1, pp.getReplicaPlacements().size()); + ReplicaPlacement rp = pp.getReplicaPlacements().iterator().next(); + assertEquals(hasExistingCollection ? liveNodes.get(1) : liveNodes.get(0), rp.getNode()); + } + + /** + * Tests that existing collection replicas are taken into account when preventing more than one + * replica per shard to be placed on any node. + */ + @Test + public void testMultiplePlacementsWithExistingReplicas() throws Exception { + String collectionName = "existingCollection"; + + // Cluster nodes and their attributes + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(3); + + // The collection already exists with shards and replicas + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName); + // Note that the collection as defined below is in a state that would NOT be returned by the + // placement plugin: shard 1 has two replicas on node 0. The plugin should still be able to + // place additional replicas as long as they don't break the rules. + List> shardsReplicas = + List.of( + List.of("NRT 0"), // shard 1 + List.of("NRT 1"), // shard 2 + List.of("NRT 2")); // shard 3 + collectionBuilder.customCollectionSetup(shardsReplicas, clusterBuilder.getLiveNodeBuilders()); + clusterBuilder.addCollection(collectionBuilder); + SolrCollection solrCollection = collectionBuilder.build(); + + List liveNodes = clusterBuilder.buildLiveNodes(); + + // Place an additional NRT and an additional TLOG replica for each shard + PlacementRequestImpl placementRequest = + new PlacementRequestImpl( + solrCollection, + solrCollection.getShardNames(), + new HashSet<>(liveNodes), + ReplicaCount.of(1, 0, 0)); + + List placementRequests = + solrCollection.getShardNames().stream() + .map( + shard -> + new PlacementRequestImpl( + solrCollection, + Collections.singleton(shard), + new HashSet<>(liveNodes), + ReplicaCount.of(1, 0, 0))) + .collect(Collectors.toList()); + // Randomize the order of the requests + Collections.shuffle(placementRequests, random()); + + // The replicas must be placed on the most appropriate nodes, i.e. those that do not already + // have a replica for the shard and then on the node with the lowest number of cores. NRT are + // placed first and given the cluster state here the placement is deterministic (easier to test, + // only one good placement). + List pps = + plugin.computePlacements(placementRequests, clusterBuilder.buildPlacementContext()); + + List replicaPlacements = + pps.stream().flatMap(pp -> pp.getReplicaPlacements().stream()).collect(Collectors.toList()); + replicaPlacements.forEach( + rp -> + assertNotEquals( + "Two replicas of the same shard on the same node", + solrCollection.getShard(rp.getShardName()).iterator().next().getNode(), + rp.getNode())); + Map> placementsByNode = + replicaPlacements.stream() + .collect(Collectors.groupingBy(ReplicaPlacement::getNode, Collectors.toList())); + for (Map.Entry> entry : placementsByNode.entrySet()) { + assertEquals( + String.format( + Locale.ROOT, + "Node %s has multiple replicas placed on it, when each node should have 1 placement: [%s]", + entry.getKey().getName(), + entry.getValue().stream() + .map(ReplicaPlacement::getShardName) + .collect(Collectors.joining(", "))), + 1, + entry.getValue().size()); + } + } + + /** + * Tests that existing collection replicas are taken into account when preventing more than one + * replica per shard to be placed on any node. + */ + @Test + public void testPlacementWithExistingReplicas() throws Exception { + String collectionName = "existingCollection"; + + // Cluster nodes and their attributes + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(5); + + // The collection already exists with shards and replicas + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName); + // Note that the collection as defined below is in a state that would NOT be returned by the + // placement plugin: shard 1 has two replicas on node 0. The plugin should still be able to + // place additional replicas as long as they don't break the rules. + List> shardsReplicas = + List.of( + List.of("NRT 0", "TLOG 0", "NRT 3"), // shard 1 + List.of("NRT 1", "NRT 3", "TLOG 2")); // shard 2 + collectionBuilder.customCollectionSetup(shardsReplicas, clusterBuilder.getLiveNodeBuilders()); + clusterBuilder.addCollection(collectionBuilder); + SolrCollection solrCollection = collectionBuilder.build(); + + List liveNodes = clusterBuilder.buildLiveNodes(); + + // Place an additional NRT and an additional TLOG replica for each shard + PlacementRequestImpl placementRequest = + new PlacementRequestImpl( + solrCollection, + solrCollection.getShardNames(), + new HashSet<>(liveNodes), + ReplicaCount.of(1, 1, 0)); + + // The replicas must be placed on the most appropriate nodes, i.e. those that do not already + // have a replica for the shard and then on the node with the lowest number of cores. NRT are + // placed first and given the cluster state here the placement is deterministic (easier to test, + // only one good placement). + PlacementPlan pp = + plugin.computePlacement(placementRequest, clusterBuilder.buildPlacementContext()); + + // Each expected placement is represented as a string "shard replica-type node" + Set expectedPlacements = Set.of("1 NRT 1", "1 TLOG 2", "2 TLOG 0", "2 NRT 4"); + verifyPlacements(expectedPlacements, pp, collectionBuilder.getShardBuilders(), liveNodes); + } + + /** + * Tests that if a collection has replicas on nodes not currently live, placement for new replicas + * works ok. + */ + @Test + public void testCollectionOnDeadNodes() throws Exception { + String collectionName = "walkingDead"; + + // Cluster nodes and their attributes + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(3); + + // The collection already exists with shards and replicas + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder(collectionName); + // The collection below has shard 1 having replicas only on dead nodes and shard 2 no replicas + // at all... (which is likely a challenging condition to recover from, but the placement + // computations should still execute happily). + List> shardsReplicas = + List.of( + List.of("NRT 10", "TLOG 11"), // shard 1 + List.of()); // shard 2 + collectionBuilder.customCollectionSetup(shardsReplicas, clusterBuilder.getLiveNodeBuilders()); + clusterBuilder.addCollection(collectionBuilder); + SolrCollection solrCollection = collectionBuilder.build(); + + List liveNodes = clusterBuilder.buildLiveNodes(); + + // Place an additional PULL replica for shard 1 + PlacementRequestImpl placementRequest = + new PlacementRequestImpl( + solrCollection, + Set.of(solrCollection.iterator().next().getShardName()), + new HashSet<>(liveNodes), + ReplicaCount.of(0, 0, 1)); + + PlacementPlan pp = + plugin.computePlacement(placementRequest, clusterBuilder.buildPlacementContext()); + + // Each expected placement is represented as a string "shard replica-type node" + // Node 0 has fewer cores than node 1 (0 vs 1) so the placement should go there. + Set expectedPlacements = Set.of("1 PULL 0"); + verifyPlacements(expectedPlacements, pp, collectionBuilder.getShardBuilders(), liveNodes); + + // If we placed instead a replica for shard 2 (starting with the same initial cluster state, not + // including the first placement above), it should go too to node 0 since it has fewer cores... + Iterator it = solrCollection.iterator(); + it.next(); // skip first shard to do placement for the second one... + placementRequest = + new PlacementRequestImpl( + solrCollection, + Set.of(it.next().getShardName()), + new HashSet<>(liveNodes), + ReplicaCount.of(0, 0, 1)); + pp = plugin.computePlacement(placementRequest, clusterBuilder.buildPlacementContext()); + expectedPlacements = Set.of("2 PULL 0"); + verifyPlacements(expectedPlacements, pp, collectionBuilder.getShardBuilders(), liveNodes); + } + + /** Tests replica balancing across all nodes in a cluster */ + @Test + public void testBalancing() throws Exception { + // Cluster nodes and their attributes + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(5); + List nodeBuilders = clusterBuilder.getLiveNodeBuilders(); + + // The collection already exists with shards and replicas + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder("a"); + // Note that the collection as defined below is in a state that would NOT be returned by the + // placement plugin: shard 1 has two replicas on node 0. The plugin should still be able to + // place additional replicas as long as they don't break the rules. + List> shardsReplicas = + List.of( + List.of("NRT 0", "TLOG 0", "NRT 3"), // shard 1 + List.of("NRT 1", "NRT 3", "TLOG 2")); // shard 2 + collectionBuilder.customCollectionSetup(shardsReplicas, nodeBuilders); + clusterBuilder.addCollection(collectionBuilder); + + BalanceRequestImpl balanceRequest = + new BalanceRequestImpl(new HashSet<>(clusterBuilder.buildLiveNodes())); + BalancePlan balancePlan = + plugin.computeBalancing(balanceRequest, clusterBuilder.buildPlacementContext()); + + // Each expected placement is represented as a string "col shard replica-type fromNode -> + // toNode" + Set expectedPlacements = Set.of("a 1 NRT 0 -> 4"); + verifyBalancing( + expectedPlacements, + balancePlan, + collectionBuilder.getShardBuilders(), + clusterBuilder.buildLiveNodes()); + } + + /** Tests that balancing works across a subset of nodes */ + @Test + public void testBalancingWithNodeSubset() throws Exception { + // Cluster nodes and their attributes + Builders.ClusterBuilder clusterBuilder = Builders.newClusterBuilder().initializeLiveNodes(5); + // The collection already exists with shards and replicas + Builders.CollectionBuilder collectionBuilder = Builders.newCollectionBuilder("a"); + // Note that the collection as defined below is in a state that would NOT be returned by the + // placement plugin: shard 1 has two replicas on node 0. The plugin should still be able to + // place additional replicas as long as they don't break the rules. + List> shardsReplicas = + List.of( + List.of("NRT 0", "TLOG 0", "NRT 3"), // shard 1 + List.of("NRT 1", "NRT 3", "TLOG 2")); // shard 2 + collectionBuilder.customCollectionSetup(shardsReplicas, clusterBuilder.getLiveNodeBuilders()); + clusterBuilder.addCollection(collectionBuilder); + + // Only balance over node 1-4 + List overNodes = clusterBuilder.buildLiveNodes(); + overNodes.remove(0); + + BalanceRequestImpl balanceRequest = new BalanceRequestImpl(new HashSet<>(overNodes)); + BalancePlan balancePlan = + plugin.computeBalancing(balanceRequest, clusterBuilder.buildPlacementContext()); + + // Each expected placement is represented as a string "col shard replica-type fromNode -> + // toNode" + Set expectedPlacements = Set.of("a 1 NRT 3 -> 4"); + verifyBalancing( + expectedPlacements, + balancePlan, + collectionBuilder.getShardBuilders(), + clusterBuilder.buildLiveNodes()); + } +} From 0c2b1a6dd615b4fc160ea2e715031c057ba3df5e Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Tue, 18 Jun 2024 14:40:52 -0700 Subject: [PATCH 003/172] SOLR-17255: Fix bugs in SolrParams.toLocalParamsString() --- solr/CHANGES.txt | 2 + .../solr/client/solrj/util/ClientUtils.java | 10 +++-- .../apache/solr/common/params/SolrParams.java | 13 ++++-- .../solr/common/params/SolrParamTest.java | 42 +++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 03876f1a414..7164de994ff 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -165,6 +165,8 @@ Bug Fixes to the `` element in `solrconfig.xml`, which has long been silently ignored, will now be respected (Michael Gibney, David Smiley) +* SOLR-17255: Fix bugs in SolrParams.toLocalParamsString() (hossman) + Dependency Upgrades --------------------- (No changes) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 16f27410400..d0cc138a8ad 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -246,14 +246,16 @@ public static String escapeQueryChars(String s) { } /** - * Returns the value encoded properly so it can be appended after a + * Returns the (literal) value encoded properly so it can be appended after a name= + * local-param key. * - * 

name=
- * - * local-param. + *

NOTE: This method assumes $ is a literal character that must be quoted. (It + * does not assume strings starting $ should be treated as param refrenes) */ public static String encodeLocalParamVal(String val) { int len = val.length(); + if (0 == len) return "''"; // quoted empty string + int i = 0; if (len > 0 && val.charAt(0) != '$') { for (; i < len; i++) { diff --git a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java index 2d2fbe9d837..a06933ed099 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java @@ -569,9 +569,9 @@ public String toQueryString() { } /** - * Generates a local-params string of the form - * - * 

{! name=value name2=value2}
+ * Generates a local-params string of the form {! name=value name2=value2}, + * Protecting (with out any quoting or escaping) any values that start with $ (param + * references). */ public String toLocalParamsString() { final StringBuilder sb = new StringBuilder(128); @@ -583,7 +583,12 @@ public String toLocalParamsString() { sb.append(' '); // do so even the first time; why not. sb.append(name); // no escaping for name; it must follow "Java Identifier" rules. sb.append('='); - sb.append(ClientUtils.encodeLocalParamVal(val)); + if (val.startsWith("$")) { + // maintain literal param ref... + sb.append(val); + } else { + sb.append(ClientUtils.encodeLocalParamVal(val)); + } } } sb.append('}'); diff --git a/solr/solrj/src/test/org/apache/solr/common/params/SolrParamTest.java b/solr/solrj/src/test/org/apache/solr/common/params/SolrParamTest.java index 6ae0e3edd4f..84b8ea1894c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/params/SolrParamTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/params/SolrParamTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.common.params; +import static org.apache.solr.SolrTestCaseJ4.params; + import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.HashMap; @@ -24,6 +26,7 @@ import java.util.Map; import org.apache.solr.SolrTestCase; import org.apache.solr.common.SolrException; +import org.apache.solr.search.QueryParsing; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +34,45 @@ public class SolrParamTest extends SolrTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public void testLocalParamRoundTripParsing() throws Exception { + final SolrParams in = + params( + "simple", "xxx", + "blank", "", + "space", "x y z", + "lead_space", " x", + "curly", "x}y", + "quote", "x'y", + "quoted", "'x y'", + "d_quote", "x\"y", + "d_quoted", "\"x y\"", + "dollar", "x$y", + "multi", "x", + "multi", "y y", + "v", "$ref"); + final String toStr = in.toLocalParamsString(); + final SolrParams out = QueryParsing.getLocalParams(toStr, params("ref", "ref value")); + + assertEquals("xxx", out.get("simple")); + assertEquals("", out.get("blank")); + assertEquals("x y z", out.get("space")); + assertEquals(" x", out.get("lead_space")); + assertEquals("x}y", out.get("curly")); + assertEquals("x'y", out.get("quote")); + assertEquals("'x y'", out.get("quoted")); + assertEquals("x\"y", out.get("d_quote")); + assertEquals("\"x y\"", out.get("d_quoted")); + assertEquals("x$y", out.get("dollar")); + + assertArrayEquals(new String[] {"x", "y y"}, out.getParams("multi")); + // first one should win... + assertEquals("x", out.get("multi")); + + assertEquals("ref value", out.get("v")); + + assertIterSize(toStr, 12, out); + } + public void testParamIterators() { ModifiableSolrParams aaa = new ModifiableSolrParams(); From 4a12f8a4b58eb01e59bd8b50b44912aa3167b6d0 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 19 Jun 2024 07:22:02 -0400 Subject: [PATCH 004/172] fix MoveReplicaHdfsFailoverTest after SOLR-16962 (#2525) the changes made by SOLR-16962 to ZkController and MoveReplicaCmd resolve the conflation of ulog and tlog dir, and remove some related hacky compensations for this conflation. The nightly test MoveReplicaHdfsFailoverTest was still compensating for this issue when it no longer needs to, and was thus failing. --- .../org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java index d6500b3f87e..f80f3242130 100644 --- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java +++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/MoveReplicaHdfsFailoverTest.java @@ -91,7 +91,6 @@ public void testDataDirAndUlogAreMaintained() throws Exception { .setNode(cluster.getJettySolrRunner(0).getNodeName()) .process(cluster.getSolrClient()); - ulogDir += "/tlog"; ZkStateReader zkStateReader = cluster.getZkStateReader(); assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000)); From 0a6f79dcda3c60e8aeadad418b8b6340f21ff6b5 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 20 Jun 2024 13:00:47 -0400 Subject: [PATCH 005/172] Remove 'IndexUpgraderTool' page from ref-guide (#2500) The IndexUpgraderTool doesn't have a clear use case for Solr users, and after some discussion on the mailing list we decided documenting it isn't worth the confusion it causes for users. This commit removes the related page from our ref-guide. --- .../deployment-guide/deployment-nav.adoc | 1 - .../pages/indexupgrader-tool.adoc | 53 ------------------- .../pages/upgrading-a-solr-cluster.adoc | 6 ++- .../pages/major-changes-in-solr-7.adoc | 1 - 4 files changed, 5 insertions(+), 56 deletions(-) delete mode 100644 solr/solr-ref-guide/modules/deployment-guide/pages/indexupgrader-tool.adoc diff --git a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc index 5ba4dd59fb0..cdf02d39dde 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc @@ -25,7 +25,6 @@ ** xref:taking-solr-to-production.adoc[] ** xref:jvm-settings.adoc[] ** xref:upgrading-a-solr-cluster.adoc[] -*** xref:indexupgrader-tool.adoc[] ** xref:backup-restore.adoc[] ** xref:solr-in-docker.adoc[] *** xref:docker-faq.adoc[] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/indexupgrader-tool.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/indexupgrader-tool.adoc deleted file mode 100644 index e6aa84ff1da..00000000000 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/indexupgrader-tool.adoc +++ /dev/null @@ -1,53 +0,0 @@ -= IndexUpgraderTool -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -The Lucene distribution includes {lucene-javadocs}/core/org/apache/lucene/index/IndexUpgrader.html[a tool that upgrades] an index from the previous Lucene version to the current file format. - -The tool can be used from command line, or it can be instantiated and executed in Java. - -[IMPORTANT] -==== -Indexes can *only* be upgraded from the previous major release version to the current major release version. - -This means that the IndexUpgraderTool in any Solr 8.x release, for example, can only work with indexes from 7.x releases, but cannot work with indexes from Solr 6.x or earlier. - -If you are currently using a release two or more major versions older, such as moving from Solr 6x to Solr 8x, you will need to reindex your content. - -The IndexUpgraderTool performs a forceMerge (optimize) down to one segment, which may be undesirable. -==== - -In a Solr distribution, the Lucene files are located in `./server/solr-webapp/webapp/WEB-INF/lib`. -You will need to include the `lucene-core-.jar` and `lucene-backwards-codecs-.jar` on the classpath when running the tool. - -[source,bash,subs="attributes"] ----- -java -cp lucene-core-{dep-version-lucene}.jar:lucene-backward-codecs-{dep-version-lucene}.jar org.apache.lucene.index.IndexUpgrader [-delete-prior-commits] [-verbose] /path/to/index ----- - -This tool keeps only the last commit in an index. -For this reason, if the incoming index has more than one commit, the tool refuses to run by default. -Specify `-delete-prior-commits` to override this, allowing the tool to delete all but the last commit. - -Upgrading large indexes may take a long time. -As a rule of thumb, the upgrade processes about 1 GB per minute. - -[WARNING] -==== -This tool may reorder documents if the index was partially upgraded before execution (e.g., documents were added). -If your application relies on monotonicity of document IDs (i.e., the order in which the documents were added to the index is preserved), do a full optimize instead. -==== diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc index c3d58219342..a2efff5a5aa 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc @@ -1,5 +1,4 @@ = Upgrading a Solr Cluster -:page-children: indexupgrader-tool // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -22,6 +21,11 @@ This page covers how to upgrade an existing Solr cluster that was installed usin IMPORTANT: The steps outlined on this page assume you use the default service name of `solr`. If you use an alternate service name or Solr installation directory, some of the paths and commands mentioned below will have to be modified accordingly. +NOTE: Previous versions of this reference guide mentioned an {lucene-javadocs}/core/org/apache/lucene/index/IndexUpgrader.html[IndexUpgraderTool] that is made available by the Lucene project, and that can be used to upgrade Solr index files offline if desired. +Users with a business requirement that necessitates its use may use this tool as before, however it's not needed in the vast majority of upgrade workflows. +The documentation below assumes the tool is not in use. + + == Planning Your Upgrade Here is a checklist of things you need to prepare before starting the upgrade process: diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-7.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-7.adoc index b1f726a9288..1aab84288c9 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-7.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-7.adoc @@ -33,7 +33,6 @@ However, if reindexing is not feasible, keep in mind you can only upgrade one ma Thus, Solr 6.x indexes will be compatible with Solr 7 but Solr 5.x indexes will not be. If you do not reindex now, keep in mind that you will need to either reindex your data or upgrade your indexes before you will be able to move to Solr 8 when it is released in the future. -See the section xref:deployment-guide:indexupgrader-tool.adoc[] for more details on how to upgrade your indexes. See also the section xref:deployment-guide:upgrading-a-solr-cluster.adoc[] for details on how to upgrade a SolrCloud cluster. From 2a56bfc2d8b48d1c93d683515569e2e97acf2077 Mon Sep 17 00:00:00 2001 From: Sanjay Dutt Date: Fri, 21 Jun 2024 20:24:03 +0530 Subject: [PATCH 006/172] SOLR-17321: Remove Deprecated URL ctors in Preparation for Java 21 (#2501) * Switch to URI * Added URL to the forbidden API ------------- Co-authored-by: Uwe Schindler Co-authored-by: David Smiley --- .../forbidden-apis/defaults.all.txt | 3 + solr/CHANGES.txt | 4 ++ .../java/org/apache/solr/cli/PostTool.java | 60 +++++++++---------- .../org/apache/solr/cli/RunExampleTool.java | 6 +- .../src/java/org/apache/solr/cli/SolrCLI.java | 3 +- .../java/org/apache/solr/cli/StatusTool.java | 4 +- .../handler/admin/AdminHandlersProxy.java | 6 +- .../DefaultPackageRepository.java | 7 ++- .../packagemanager/RepositoryManager.java | 4 +- .../schema/OpenExchangeRatesOrgProvider.java | 4 +- .../solr/security/AllowListUrlChecker.java | 8 +-- .../apache/solr/security/SolrNodeKeyPair.java | 5 +- .../solr/servlet/SolrRequestParsers.java | 4 +- .../processor/URLClassifyProcessor.java | 2 +- .../org/apache/solr/cli/PostToolTest.java | 38 +++++++----- .../apache/solr/cloud/OverseerRolesTest.java | 4 +- .../cloud/TestPullReplicaErrorHandling.java | 19 +++--- .../solr/cloud/TestRequestForwarding.java | 10 +++- .../solr/handler/ReplicationTestHelper.java | 5 +- .../solr/handler/TestReplicationHandler.java | 3 +- .../handler/TestReplicationHandlerBackup.java | 3 +- .../solr/request/TestRemoteStreaming.java | 4 +- .../TestSubQueryTransformerDistrib.java | 14 +++-- .../security/AuditLoggerIntegrationTest.java | 4 +- .../solr/servlet/HttpSolrCallCloudTest.java | 5 +- .../solr/security/jwt/JWTIssuerConfig.java | 7 ++- .../jwt/JWTAuthPluginIntegrationTest.java | 7 ++- .../solr/client/solrj/util/ClientUtils.java | 8 +-- .../client/solrj/TestSolrJErrorHandling.java | 3 +- .../solrj/embedded/JettyWebappTest.java | 4 +- .../solrj/impl/CloudHttp2SolrClientTest.java | 6 +- .../solrj/impl/CloudSolrClientTest.java | 6 +- .../solr/common/util/ContentStreamTest.java | 13 ++-- .../apache/solr/embedded/JettySolrRunner.java | 14 ++++- .../solr/handler/BackupRestoreUtils.java | 3 +- .../solr/handler/TestRestoreCoreUtil.java | 3 +- 36 files changed, 168 insertions(+), 135 deletions(-) diff --git a/gradle/validation/forbidden-apis/defaults.all.txt b/gradle/validation/forbidden-apis/defaults.all.txt index a7647ddbcb7..f1220bfcd05 100644 --- a/gradle/validation/forbidden-apis/defaults.all.txt +++ b/gradle/validation/forbidden-apis/defaults.all.txt @@ -83,3 +83,6 @@ java.util.logging.** @defaultMessage Use List.sort(Comparator) instead of Collections.sort(List, Comparator) please. java.util.Collections#sort(java.util.List, java.util.Comparator) + +@defaultMessage Use URI.toURL() to construct an instance of URL. +java.net.URL#(**) \ No newline at end of file diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7164de994ff..ca642e9bbc3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -182,6 +182,10 @@ Other Changes * SOLR-16503: Use Jetty HTTP2 for SyncStrategy and PeerSyncWithLeader for "recovery" operations (Sanjay Dutt, David Smiley) +* SOLR-16796: Include cyclonedx SBOMs with maven artifacts (Arnout Engelen, Houston Putman, Kevin Risden) + +* SOLR-17321: Remove Deprecated URL and replace it with URI in Preparation for Java 21 (Sanjay Dutt, David Smiley, Uwe Schindler) + ================== 9.6.1 ================== Bug Fixes --------------------- diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java index 45eb73c6408..1181910b2a4 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java @@ -98,7 +98,7 @@ public class PostTool extends ToolBase { int recursive = 0; int delay = 0; String fileTypes = PostTool.DEFAULT_FILE_TYPES; - URL solrUpdateUrl; + URI solrUpdateUrl; String credentials; OutputStream out = null; String type; @@ -256,10 +256,10 @@ public void runImpl(CommandLine cli) throws Exception { solrUpdateUrl = null; if (cli.hasOption("url")) { String url = cli.getOptionValue("url"); - solrUpdateUrl = new URL(url); + solrUpdateUrl = new URI(url); } else if (cli.hasOption("c")) { String url = SolrCLI.getDefaultSolrUrl() + "/solr/" + cli.getOptionValue("c") + "/update"; - solrUpdateUrl = new URL(url); + solrUpdateUrl = new URI(url); } else { throw new IllegalArgumentException( "Must specify either -url or -c parameter to post documents."); @@ -389,7 +389,7 @@ private void doWebMode() { numPagesPosted = postWebPages(args, 0, out); info(numPagesPosted + " web pages indexed."); - } catch (MalformedURLException e) { + } catch (URISyntaxException e) { warn("Wrong URL trying to append /extract to " + solrUpdateUrl); } } @@ -515,7 +515,7 @@ int postFiles(File[] files, OutputStream out, String type) { postFile(srcFile, out, type); Thread.sleep(delay * 1000L); filesPosted++; - } catch (InterruptedException | MalformedURLException e) { + } catch (InterruptedException | URISyntaxException e) { throw new RuntimeException(e); } } @@ -623,8 +623,8 @@ protected int webCrawl(int level, OutputStream out) { PostTool.PageFetcherResult result = pageFetcher.readPageFromUrl(url); if (result.httpStatus == 200) { url = (result.redirectUrl != null) ? result.redirectUrl : url; - URL postUrl = - new URL( + URI postUri = + new URI( appendParam( solrUpdateUrl.toString(), "literal.id=" @@ -638,7 +638,7 @@ protected int webCrawl(int level, OutputStream out) { null, out, result.contentType, - postUrl); + postUri); if (success) { info("POSTed web resource " + url + " (depth: " + level + ")"); Thread.sleep(delay * 1000L); @@ -651,7 +651,7 @@ protected int webCrawl(int level, OutputStream out) { new ByteArrayInputStream( content.array(), content.arrayOffset(), content.limit()), result.contentType, - postUrl); + postUri); subStack.addAll(children); } } else { @@ -782,10 +782,10 @@ public static String appendParam(String url, String param) { } /** Opens the file and posts its contents to the solrUrl, writes to response to output. */ - public void postFile(File file, OutputStream output, String type) throws MalformedURLException { + public void postFile(File file, OutputStream output, String type) throws URISyntaxException { InputStream is = null; - URL url = solrUpdateUrl; + URI uri = solrUpdateUrl; String suffix = ""; if (auto) { if (type == null) { @@ -797,7 +797,7 @@ public void postFile(File file, OutputStream output, String type) throws Malform if (type.equals("application/json") && !PostTool.FORMAT_SOLR.equals(format)) { suffix = "/json/docs"; String urlStr = appendUrlPath(solrUpdateUrl, suffix).toString(); - url = new URL(urlStr); + uri = new URI(urlStr); } else if (type.equals("application/xml") || type.equals("text/csv") || type.equals("application/json")) { @@ -815,7 +815,7 @@ public void postFile(File file, OutputStream output, String type) throws Malform urlStr = appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), UTF_8)); } - url = new URL(urlStr); + uri = new URI(urlStr); } } else { if (type == null) { @@ -838,7 +838,7 @@ public void postFile(File file, OutputStream output, String type) throws Malform + " to [base]" + suffix); is = new FileInputStream(file); - postData(is, file.length(), output, type, url); + postData(is, file.length(), output, type, uri); } catch (IOException e) { warn("Can't open/read file: " + file); } finally { @@ -856,18 +856,13 @@ public void postFile(File file, OutputStream output, String type) throws Malform /** * Appends to the path of the URL * - * @param url the URL + * @param uri the URI * @param append the path to append * @return the final URL version */ - protected static URL appendUrlPath(URL url, String append) throws MalformedURLException { - return new URL( - url.getProtocol() - + "://" - + url.getAuthority() - + url.getPath() - + append - + (url.getQuery() != null ? "?" + url.getQuery() : "")); + protected static URI appendUrlPath(URI uri, String append) throws URISyntaxException { + var newPath = uri.getPath() + append; + return new URI(uri.getScheme(), uri.getAuthority(), newPath, uri.getQuery(), uri.getFragment()); } /** @@ -890,7 +885,7 @@ protected static String guessType(File file) { * @return true if success */ public boolean postData( - InputStream data, Long length, OutputStream output, String type, URL url) { + InputStream data, Long length, OutputStream output, String type, URI uri) { if (dryRun) { return true; } @@ -902,7 +897,7 @@ public boolean postData( HttpURLConnection urlConnection = null; try { try { - urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection = (HttpURLConnection) (uri.toURL()).openConnection(); try { urlConnection.setRequestMethod("POST"); } catch (ProtocolException e) { @@ -1200,9 +1195,9 @@ public boolean isDisallowedByRobots(URL url) { disallows = new ArrayList<>(); URL urlRobot; try { - urlRobot = new URL(strRobot); + urlRobot = new URI(strRobot).toURL(); disallows = parseRobotsTxt(urlRobot.openStream()); - } catch (MalformedURLException e) { + } catch (URISyntaxException | MalformedURLException e) { return true; // We cannot trust this robots URL, should not happen } catch (IOException e) { // There is no robots.txt, will cache an empty disallow list @@ -1250,17 +1245,16 @@ protected List parseRobotsTxt(InputStream is) throws IOException { * @param url the URL of the web page * @param is the input stream of the page * @param type the content-type - * @param postUrl the URL (typically /solr/extract) in order to pull out links + * @param postUri the URI (typically /solr/extract) in order to pull out links * @return a set of URIs parsed from the page */ - protected Set getLinksFromWebPage(URL url, InputStream is, String type, URL postUrl) { + protected Set getLinksFromWebPage(URL url, InputStream is, String type, URI postUri) { Set linksFromPage = new HashSet<>(); try { ByteArrayOutputStream os = new ByteArrayOutputStream(); - URL extractUrl = new URL(appendParam(postUrl.toString(), "extractOnly=true")); - extractUrl = new URL(appendParam(extractUrl.toString(), "wt=xml")); - boolean success = postData(is, null, os, type, extractUrl); + URI extractUri = new URI(appendParam(postUri.toString(), "extractOnly=true")); + boolean success = postData(is, null, os, type, extractUri); if (success) { Document d = makeDom(os.toByteArray()); String innerXml = getXP(d, "/response/str/text()[1]", false); @@ -1279,7 +1273,7 @@ protected Set getLinksFromWebPage(URL url, InputStream is, String type, URL } } } - } catch (MalformedURLException e) { + } catch (URISyntaxException e) { warn("Malformed URL " + url); } catch (IOException e) { warn("IOException opening URL " + url + ": " + e.getMessage()); diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java index c4e5e78ecde..73ff1dd8772 100644 --- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java +++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java @@ -22,7 +22,7 @@ import java.io.InputStream; import java.io.PrintStream; import java.net.Socket; -import java.net.URL; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Arrays; @@ -890,7 +890,7 @@ protected Map getNodeStatus(String solrUrl, String credentials, StatusTool statusTool = new StatusTool(); if (verbose) echo("\nChecking status of Solr at " + solrUrl + " ..."); - URL solrURL = new URL(solrUrl); + URI solrURI = new URI(solrUrl); Map nodeStatus = statusTool.waitToSeeSolrUp(solrUrl, credentials, maxWaitSecs, TimeUnit.SECONDS); nodeStatus.put("baseUrl", solrUrl); @@ -900,7 +900,7 @@ protected Map getNodeStatus(String solrUrl, String credentials, if (verbose) echo( "\nSolr is running on " - + solrURL.getPort() + + solrURI.getPort() + " in " + mode + " mode with status:\n" diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index 191fb48bda4..93fdbf3697c 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.SocketException; +import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -359,7 +360,7 @@ private static Set findClasses(String path, String packageName) throws E Set classes = new TreeSet<>(); if (path.startsWith("file:") && path.contains("!")) { String[] split = path.split("!"); - URL jar = new URL(split[0]); + URL jar = new URI(split[0]).toURL(); try (ZipInputStream zip = new ZipInputStream(jar.openStream())) { ZipEntry entry; while ((entry = zip.getNextEntry()) != null) { diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/cli/StatusTool.java index a201fc42641..840539c69c4 100644 --- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java +++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java @@ -18,7 +18,7 @@ package org.apache.solr.cli; import java.io.PrintStream; -import java.net.URL; +import java.net.URI; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -94,7 +94,7 @@ public void runImpl(CommandLine cli) throws Exception { int maxWaitSecs = Integer.parseInt(cli.getOptionValue("maxWaitSecs", "0")); String solrUrl = SolrCLI.normalizeSolrUrl(cli); if (maxWaitSecs > 0) { - int solrPort = (new URL(solrUrl)).getPort(); + int solrPort = new URI(solrUrl).getPort(); echo("Waiting up to " + maxWaitSecs + " seconds to see Solr running on port " + solrPort); try { waitToSeeSolrUp( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java index cd278fd095a..2d5b8d9f6f2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -133,8 +133,8 @@ public static Pair>, SolrClient> callRemoteNode( String nodeName, String endpoint, SolrParams params, ZkController zkController) throws IOException, SolrServerException { log.debug("Proxying {} request to node {}", endpoint, nodeName); - URL baseUrl = new URL(zkController.zkStateReader.getBaseUrlForNodeName(nodeName)); - HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build(); + URI baseUri = URI.create(zkController.zkStateReader.getBaseUrlForNodeName(nodeName)); + HttpSolrClient solr = new HttpSolrClient.Builder(baseUri.toString()).build(); SolrRequest proxyReq = new GenericSolrRequest(SolrRequest.METHOD.GET, endpoint, params); HttpSolrClient.HttpUriRequestResponse proxyResp = solr.httpUriRequest(proxyReq); return new Pair<>(proxyResp.future, solr); diff --git a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java index 820391804e1..62d241d8bce 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -86,9 +87,9 @@ public Path download(String artifactName) throws SolrException, IOException { Path tmpDirectory = Files.createTempDirectory("solr-packages"); tmpDirectory.toFile().deleteOnExit(); URL url = - new URL( - new URL(repositoryURL.endsWith("/") ? repositoryURL : repositoryURL + "/"), - artifactName); + URI.create(repositoryURL.endsWith("/") ? repositoryURL : repositoryURL + "/") + .resolve(artifactName) + .toURL(); String fileName = FilenameUtils.getName(url.getPath()); Path destination = tmpDirectory.resolve(fileName); diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 4e3be7b16c6..09c0a209172 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -24,7 +24,7 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -133,7 +133,7 @@ public void addRepository(String repoName, String uri) throws Exception { true); } - try (InputStream is = new URL(uri + "/publickey.der").openStream()) { + try (InputStream is = new URI(uri + "/publickey.der").toURL().openStream()) { addKey(is.readAllBytes(), repoName + ".der"); } } diff --git a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java index 6438af1ffe6..fcc64b111eb 100644 --- a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java +++ b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java @@ -20,7 +20,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -151,7 +151,7 @@ public boolean reload() throws SolrException { try { log.debug("Reloading exchange rates from {}", ratesFileLocation); try { - ratesJsonStream = (new URL(ratesFileLocation)).openStream(); + ratesJsonStream = (new URI(ratesFileLocation).toURL()).openStream(); } catch (Exception e) { ratesJsonStream = resourceLoader.openResource(ratesFileLocation); } diff --git a/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java b/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java index 1354fde0ec7..9bcede9b060 100644 --- a/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java +++ b/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java @@ -20,7 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -191,16 +191,16 @@ private static String parseHostPort(String url) throws MalformedURLException { // Parse the host and port. // It doesn't really matter which protocol we set here because we are not going to use it. url = url.trim(); - URL u; + URI u; Matcher protocolMatcher = PROTOCOL_PATTERN.matcher(url); if (protocolMatcher.matches()) { // Replace any protocol unsupported by URL. if (!protocolMatcher.group(1).startsWith("http")) { url = "http" + protocolMatcher.group(2); } - u = new URL(url); + u = URI.create(url); } else { - u = new URL("http://" + url); + u = URI.create("http://" + url); } if (u.getHost() == null || u.getPort() < 0) { throw new MalformedURLException("Invalid host or port in '" + url + "'"); diff --git a/solr/core/src/java/org/apache/solr/security/SolrNodeKeyPair.java b/solr/core/src/java/org/apache/solr/security/SolrNodeKeyPair.java index 1e93667c5d9..8db42492354 100644 --- a/solr/core/src/java/org/apache/solr/security/SolrNodeKeyPair.java +++ b/solr/core/src/java/org/apache/solr/security/SolrNodeKeyPair.java @@ -18,7 +18,7 @@ package org.apache.solr.security; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.security.spec.InvalidKeySpecException; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CloudConfig; @@ -55,7 +55,8 @@ private static CryptoKeys.RSAKeyPair createKeyPair(CloudConfig config) { } try { - return new CryptoKeys.RSAKeyPair(new URL(privateKey), new URL(publicKey)); + return new CryptoKeys.RSAKeyPair( + URI.create(privateKey).toURL(), URI.create(publicKey).toURL()); } catch (IOException | InvalidKeySpecException e) { throw new RuntimeException("Bad PublicKeyHandler configuration.", e); } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index 91d7e2b2498..38a45c71a91 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -24,7 +24,7 @@ import java.io.InputStream; import java.io.PushbackInputStream; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; @@ -212,7 +212,7 @@ private SolrQueryRequest buildRequestFrom( throw new SolrException(ErrorCode.BAD_REQUEST, "Remote Streaming is disabled."); } for (final String url : strs) { - ContentStreamBase stream = new ContentStreamBase.URLStream(new URL(url)); + ContentStreamBase stream = new ContentStreamBase.URLStream(URI.create(url).toURL()); if (contentType != null) { stream.setContentType(contentType); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java index f26450442e4..91ecc1adad1 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/URLClassifyProcessor.java @@ -198,7 +198,7 @@ public URL getCanonicalUrl(URL url) throws MalformedURLException { // NOTE: Do we want to make sure this URL is normalized? (Christian thinks we should) String urlString = url.toString(); String lps = landingPageSuffix(url); - return new URL(urlString.replaceFirst("/" + lps + "$", "/")); + return URI.create(urlString.replaceFirst("/" + lps + "$", "/")).toURL(); } /** diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java index 5d8b9720f6b..a6e60f84bed 100644 --- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java @@ -171,19 +171,24 @@ public void testComputeFullUrl() throws IOException { assertEquals( "http://[ff01::114]/index.html", - webPostTool.computeFullUrl(new URL("http://[ff01::114]/"), "/index.html")); + webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "/index.html")); assertEquals( "http://[ff01::114]/index.html", - webPostTool.computeFullUrl(new URL("http://[ff01::114]/foo/bar/"), "/index.html")); + webPostTool.computeFullUrl( + URI.create("http://[ff01::114]/foo/bar/").toURL(), "/index.html")); assertEquals( "http://[ff01::114]/fil.html", - webPostTool.computeFullUrl(new URL("http://[ff01::114]/foo.htm?baz#hello"), "fil.html")); + webPostTool.computeFullUrl( + URI.create("http://[ff01::114]/foo.htm?baz#hello").toURL(), "fil.html")); // TODO: How to know what is the base if URL path ends with "foo"?? // assertEquals("http://[ff01::114]/fil.html", t_web.computeFullUrl(new // URL("http://[ff01::114]/foo?baz#hello"), "fil.html")); - assertNull(webPostTool.computeFullUrl(new URL("http://[ff01::114]/"), "fil.jpg")); - assertNull(webPostTool.computeFullUrl(new URL("http://[ff01::114]/"), "mailto:hello@foo.bar")); - assertNull(webPostTool.computeFullUrl(new URL("http://[ff01::114]/"), "ftp://server/file")); + assertNull(webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "fil.jpg")); + assertNull( + webPostTool.computeFullUrl( + URI.create("http://[ff01::114]/").toURL(), "mailto:hello@foo.bar")); + assertNull( + webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "ftp://server/file")); } @Test @@ -210,10 +215,10 @@ public void testAppendParam() { } @Test - public void testAppendUrlPath() throws MalformedURLException { + public void testAppendUrlPath() throws URISyntaxException { assertEquals( - new URL("http://[ff01::114]/a?foo=bar"), - PostTool.appendUrlPath(new URL("http://[ff01::114]?foo=bar"), "/a")); + URI.create("http://[ff01::114]/a?foo=bar"), + PostTool.appendUrlPath(URI.create("http://[ff01::114]?foo=bar"), "/a")); } @Test @@ -231,7 +236,7 @@ public void testDoFilesMode() throws MalformedURLException { PostTool postTool = new PostTool(); postTool.recursive = 0; postTool.dryRun = true; - postTool.solrUpdateUrl = new URL("http://localhost:8983/solr/fake/update"); + postTool.solrUpdateUrl = URI.create("http://localhost:8983/solr/fake/update"); File dir = getFile("exampledocs"); int num = postTool.postFiles(new String[] {dir.toString()}, 0, null, null); assertEquals(2, num); @@ -253,7 +258,7 @@ public void testRecursionAppliesToFilesMode() throws MalformedURLException { PostTool postTool = new PostTool(); postTool.recursive = 1; // This is the default postTool.dryRun = true; - postTool.solrUpdateUrl = new URL("http://localhost:8983/solr/fake/update"); + postTool.solrUpdateUrl = URI.create("http://localhost:8983/solr/fake/update"); File dir = getFile("exampledocs"); int num = postTool.postFiles(new String[] {dir.toString()}, 0, null, null); assertEquals(2, num); @@ -264,7 +269,7 @@ public void testDoWebMode() throws IOException, URISyntaxException { PostTool postTool = new PostTool(); postTool.pageFetcher = new MockPageFetcher(); postTool.dryRun = true; - postTool.solrUpdateUrl = new URL("http://user:password@localhost:5150/solr/fake/update"); + postTool.solrUpdateUrl = URI.create("http://user:password@localhost:5150/solr/fake/update"); // Uses mock pageFetcher postTool.delay = 0; @@ -289,8 +294,11 @@ public void testRobotsExclusion() throws IOException, URISyntaxException { postTool.pageFetcher = new MockPageFetcher(); postTool.dryRun = true; - assertFalse(postTool.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/"))); - assertTrue(postTool.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/disallowed"))); + assertFalse( + postTool.pageFetcher.isDisallowedByRobots(URI.create("http://[ff01::114]/").toURL())); + assertTrue( + postTool.pageFetcher.isDisallowedByRobots( + URI.create("http://[ff01::114]/disallowed").toURL())); assertEquals( "There should be two entries parsed from robots.txt", 2, @@ -368,7 +376,7 @@ public PostTool.PageFetcherResult readPageFromUrl(URL u) { } @Override - public Set getLinksFromWebPage(URL url, InputStream is, String type, URL postUrl) { + public Set getLinksFromWebPage(URL url, InputStream is, String type, URI postUri) { Set s = linkMap.get(PostTool.normalizeUrlEnding(url.toString())); if (s == null) { s = new HashSet<>(); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java index 8ebe618e660..3da6970e1f6 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java @@ -20,7 +20,7 @@ import static org.apache.solr.cloud.OverseerTaskProcessor.getSortedElectionNodes; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -75,7 +75,7 @@ public static void waitForNewOverseer( private JettySolrRunner getOverseerJetty() throws Exception { String overseer = getLeaderNode(zkClient()); - URL overseerUrl = new URL("http://" + overseer.substring(0, overseer.indexOf('_'))); + URI overseerUrl = URI.create("http://" + overseer.substring(0, overseer.indexOf('_'))); int hostPort = overseerUrl.getPort(); for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { try { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java index dd0363c16ac..9906d3041f1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.net.URL; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -325,24 +324,24 @@ private DocCollection assertNumberOfReplicas( protected JettySolrRunner getJettyForReplica(Replica replica) throws Exception { String replicaBaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP); assertNotNull(replicaBaseUrl); - URL baseUrl = new URL(replicaBaseUrl); + URI baseUri = URI.create(replicaBaseUrl); - JettySolrRunner proxy = jettys.get(baseUrl.toURI()); - assertNotNull("No proxy found for " + baseUrl + "!", proxy); + JettySolrRunner proxy = jettys.get(baseUri); + assertNotNull("No proxy found for " + baseUri + "!", proxy); return proxy; } protected SocketProxy getProxyForReplica(Replica replica) throws Exception { String replicaBaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP); assertNotNull(replicaBaseUrl); - URL baseUrl = new URL(replicaBaseUrl); + URI baseUri = URI.create(replicaBaseUrl); - SocketProxy proxy = proxies.get(baseUrl.toURI()); - if (proxy == null && !baseUrl.toExternalForm().endsWith("/")) { - baseUrl = new URL(baseUrl.toExternalForm() + "/"); - proxy = proxies.get(baseUrl.toURI()); + SocketProxy proxy = proxies.get(baseUri); + if (proxy == null && !baseUri.toString().endsWith("/")) { + baseUri = URI.create(baseUri.toString() + "/"); + proxy = proxies.get(baseUri); } - assertNotNull("No proxy found for " + baseUrl + "!", proxy); + assertNotNull("No proxy found for " + baseUri + "!", proxy); return proxy; } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRequestForwarding.java b/solr/core/src/test/org/apache/solr/cloud/TestRequestForwarding.java index a4854134f93..c672da3f4f2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRequestForwarding.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRequestForwarding.java @@ -16,11 +16,13 @@ */ package org.apache.solr.cloud; +import java.net.MalformedURLException; import java.net.URL; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; import org.junit.Test; @@ -60,7 +62,7 @@ public void testMultiCollectionQuery() throws Exception { }; for (String q : queryStrings) { try { - URL url = new URL(jettySolrRunner.getBaseUrl().toString() + "/collection1/select?" + q); + URL url = createURL(jettySolrRunner.getBaseUrl().toString() + "/collection1/select?" + q); url.openStream(); // Shouldn't throw any errors } catch (Exception ex) { throw new RuntimeException("Query '" + q + "' failed, ", ex); @@ -69,6 +71,12 @@ public void testMultiCollectionQuery() throws Exception { } } + // Restricting the Scope of Forbidden API + @SuppressForbidden(reason = "java.net.URL# deprecated since Java 20") + private URL createURL(String url) throws MalformedURLException { + return new URL(url); + } + private void createCollection(String name, String config) throws Exception { CollectionAdminResponse response; CollectionAdminRequest.Create create = diff --git a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java index a9c81429467..edd68727f87 100644 --- a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java +++ b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.io.Writer; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -156,7 +157,7 @@ public static void invokeReplicationCommand(String baseUrl, String pCommand) thr // String leaderUrl = buildUrl(pJettyPort) + "/" + DEFAULT_TEST_CORENAME + // ReplicationHandler.PATH+"?command=" + pCommand; String url = baseUrl + ReplicationHandler.PATH + "?command=" + pCommand; - URL u = new URL(url); + URL u = URI.create(url).toURL(); InputStream stream = u.openStream(); stream.close(); } @@ -227,7 +228,7 @@ public static void pullFromTo(String srcUrl, String destUrl) throws IOException + "?wait=true&command=fetchindex&leaderUrl=" + srcUrl + ReplicationHandler.PATH; - url = new URL(leaderUrl); + url = URI.create(leaderUrl).toURL(); stream = url.openStream(); stream.close(); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index c35f5c88069..8f424e97803 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.net.URL; import java.nio.file.Paths; import java.util.Arrays; @@ -844,7 +845,7 @@ public void doTestIndexFetchWithLeaderUrl() throws Exception { + "/" + DEFAULT_TEST_CORENAME + ReplicationHandler.PATH; - URL url = new URL(leaderUrl); + URL url = new URI(leaderUrl).toURL(); InputStream stream = url.openStream(); stream.close(); diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java index a85416727e6..418ce1d6551 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.net.URL; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -260,7 +261,7 @@ public static void runBackupCommand(JettySolrRunner leaderJetty, String cmd, Str + "?wt=xml&command=" + cmd + params; - URL url = new URL(leaderUrl); + URL url = URI.create(leaderUrl).toURL(); try (InputStream stream = url.openStream()) { assert stream != null; } diff --git a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java index 486b3396afd..8fb2da411d9 100644 --- a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java +++ b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java @@ -21,7 +21,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringWriter; -import java.net.URL; +import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import org.apache.lucene.tests.util.LuceneTestCase; @@ -83,7 +83,7 @@ public void testStreamUrl() throws Exception { } private String attemptHttpGet(String getUrl) throws IOException { - Object obj = new URL(getUrl).getContent(); + Object obj = URI.create(getUrl).toURL().getContent(); if (obj instanceof InputStream) { try (InputStream inputStream = (InputStream) obj) { StringWriter strWriter = new StringWriter(); diff --git a/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java b/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java index c45541a8764..5d626446e58 100644 --- a/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java +++ b/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; @@ -209,12 +210,13 @@ public void test() throws Exception { params.set("wt", "json"); final URL node = - new URL( - cluster.getRandomJetty(random()).getBaseUrl().toString() - + "/" - + people - + "/select" - + params.toQueryString()); + URI.create( + cluster.getRandomJetty(random()).getBaseUrl().toString() + + "/" + + people + + "/select" + + params.toQueryString()) + .toURL(); final URLConnection urlConnectionWithoutAuth = node.openConnection(); assertThrows(Exception.class, () -> urlConnectionWithoutAuth.getInputStream()); diff --git a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java index 47886c25380..6f800b34f54 100644 --- a/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/AuditLoggerIntegrationTest.java @@ -39,7 +39,7 @@ import java.lang.invoke.MethodHandles; import java.net.ServerSocket; import java.net.Socket; -import java.net.URL; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; @@ -257,7 +257,7 @@ public void illegalAdminPathError() throws Exception { expectThrows( FileNotFoundException.class, () -> { - try (InputStream is = new URL(baseUrl + "/node/foo").openStream()) { + try (InputStream is = URI.create(baseUrl + "/node/foo").toURL().openStream()) { new String(is.readAllBytes(), StandardCharsets.UTF_8); } }); diff --git a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java index 5af1ead66ef..7fff377d3a3 100644 --- a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java @@ -18,7 +18,7 @@ package org.apache.solr.servlet; import java.net.HttpURLConnection; -import java.net.URL; +import java.net.URI; import java.util.HashSet; import java.util.Set; import javax.servlet.ReadListener; @@ -67,7 +67,8 @@ public void testCoreChosen() throws Exception { public void testWrongUtf8InQ() throws Exception { var baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); var request = - new URL(baseUrl.toString() + "/" + COLLECTION + "/select?q=%C0"); // Illegal UTF-8 string + URI.create(baseUrl.toString() + "/" + COLLECTION + "/select?q=%C0") + .toURL(); // Illegal UTF-8 string var connection = (HttpURLConnection) request.openConnection(); assertEquals(400, connection.getResponseCode()); } diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java index 752bfd60780..7e479be74d4 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java @@ -24,6 +24,7 @@ import java.lang.invoke.MethodHandles; import java.net.InetAddress; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -118,7 +119,7 @@ public void init() { } if (wellKnownUrl != null) { try { - wellKnownDiscoveryConfig = fetchWellKnown(new URL(wellKnownUrl)); + wellKnownDiscoveryConfig = fetchWellKnown(URI.create(wellKnownUrl).toURL()); } catch (MalformedURLException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -469,7 +470,7 @@ public HttpsJwksFactory( private HttpsJwks create(String url) { final URL jwksUrl; try { - jwksUrl = new URL(url); + jwksUrl = URI.create(url).toURL(); checkAllowOutboundHttpConnections(PARAM_JWKS_URL, jwksUrl); } catch (MalformedURLException e) { throw new SolrException( @@ -506,7 +507,7 @@ public static class WellKnownDiscoveryConfig { } public static WellKnownDiscoveryConfig parse(String urlString) throws MalformedURLException { - return parse(new URL(urlString), null); + return parse(URI.create(urlString).toURL(), null); } /** diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java index ef7fe35d4f5..3bb08460218 100644 --- a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.InetAddress; +import java.net.URI; import java.net.URL; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -389,7 +390,7 @@ private void getAndFail(String url, String token) { } private Pair get(String url, String token) throws IOException { - URL createUrl = new URL(url); + URL createUrl = URI.create(url).toURL(); HttpURLConnection createConn = (HttpURLConnection) createUrl.openConnection(); if (token != null) createConn.setRequestProperty("Authorization", "Bearer " + token); BufferedReader br2 = @@ -402,7 +403,7 @@ private Pair get(String url, String token) throws IOException { } private Map getHeaders(String url, String token) throws IOException { - URL createUrl = new URL(url); + URL createUrl = URI.create(url).toURL(); HttpURLConnection conn = (HttpURLConnection) createUrl.openConnection(); if (token != null) conn.setRequestProperty("Authorization", "Bearer " + token); conn.connect(); @@ -415,7 +416,7 @@ private Map getHeaders(String url, String token) throws IOExcept } private Pair post(String url, String json, String token) throws IOException { - URL createUrl = new URL(url); + URL createUrl = URI.create(url).toURL(); HttpURLConnection con = (HttpURLConnection) createUrl.openConnection(); con.setRequestMethod("POST"); con.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index d0cc138a8ad..568bd7e5a77 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -20,7 +20,7 @@ import java.io.StringWriter; import java.io.Writer; import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -103,9 +103,9 @@ public static String buildRequestUrl( } private static String changeV2RequestEndpoint(String basePath) throws MalformedURLException { - URL oldURL = new URL(basePath); - String newPath = oldURL.getPath().replaceFirst("/solr", "/api"); - return new URL(oldURL.getProtocol(), oldURL.getHost(), oldURL.getPort(), newPath).toString(); + URI oldURI = URI.create(basePath); + String newPath = oldURI.getPath().replaceFirst("/solr", "/api"); + return oldURI.resolve(newPath).toString(); } // ------------------------------------------------------------------------ diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java index 0941775980d..68ba3e42606 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java @@ -24,6 +24,7 @@ import java.lang.invoke.MethodHandles; import java.net.HttpURLConnection; import java.net.Socket; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -280,7 +281,7 @@ public void testHttpURLConnection() throws Exception { String urlString = getCoreUrl() + "/update"; HttpURLConnection conn = null; - URL url = new URL(urlString); + URL url = URI.create(urlString).toURL(); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("POST"); diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java index 61968db2e4d..024b79b5c1d 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java @@ -18,7 +18,7 @@ import java.io.File; import java.io.InputStream; -import java.net.URL; +import java.net.URI; import java.util.Locale; import java.util.Random; import org.apache.http.Header; @@ -89,7 +89,7 @@ public void tearDown() throws Exception { public void testAdminUI() throws Exception { // Not an extensive test, but it does connect to Solr and verify the Admin ui shows up. String adminPath = "http://127.0.0.1:" + port + "/solr/"; - try (InputStream is = new URL(adminPath).openStream()) { + try (InputStream is = URI.create(adminPath).toURL().openStream()) { assertNotNull(is.readAllBytes()); // real error will be an exception } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 0b82d36a31d..474ebb8a177 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -499,8 +499,8 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // Make sure the distributed queries were directed to a single node only Set ports = new HashSet(); for (String shardAddr : shardAddresses) { - URL url = new URL(shardAddr); - ports.add(url.getPort()); + URI uri = URI.create(shardAddr); + ports.add(uri.getPort()); } // This assertion would hold true as long as every shard has a core on each node diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java index f0246b53fc8..e40e126254c 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -491,8 +491,8 @@ private void queryWithShardsPreferenceRules(CloudSolrClient cloudClient, String // Make sure the distributed queries were directed to a single node only Set ports = new HashSet(); for (String shardAddr : shardAddresses) { - URL url = new URL(shardAddr); - ports.add(url.getPort()); + URI uri = URI.create(shardAddr); + ports.add(uri.getPort()); } // This assertion would hold true as long as every shard has a core on each node diff --git a/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java b/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java index fdb5f4176c9..cc4fd1b156c 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java @@ -23,7 +23,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.zip.GZIPInputStream; @@ -101,8 +100,7 @@ public void testURLStream() throws IOException { is.transferTo(os); } - ContentStreamBase stream = - new ContentStreamBase.URLStream(new URL(file.toURI().toASCIIString())); + ContentStreamBase stream = new ContentStreamBase.URLStream(file.toURI().toURL()); try (InputStream s = stream.getStream(); FileInputStream fis = new FileInputStream(file); @@ -133,8 +131,7 @@ public void testURLStreamGZIP() throws IOException { is.transferTo(zos); } - ContentStreamBase stream = - new ContentStreamBase.URLStream(new URL(file.toURI().toASCIIString())); + ContentStreamBase stream = new ContentStreamBase.URLStream(file.toURI().toURL()); try (InputStream s = stream.getStream(); FileInputStream fis = new FileInputStream(file); GZIPInputStream zis = new GZIPInputStream(fis); @@ -160,8 +157,7 @@ public void testURLStreamCSVGZIPExtention() throws IOException { is.transferTo(zos); } - ContentStreamBase stream = - new ContentStreamBase.URLStream(new URL(file.toURI().toASCIIString())); + ContentStreamBase stream = new ContentStreamBase.URLStream(file.toURI().toURL()); try (InputStream s = stream.getStream(); FileInputStream fis = new FileInputStream(file); GZIPInputStream zis = new GZIPInputStream(fis); @@ -187,8 +183,7 @@ public void testURLStreamJSONGZIPExtention() throws IOException { is.transferTo(zos); } - ContentStreamBase stream = - new ContentStreamBase.URLStream(new URL(file.toURI().toASCIIString())); + ContentStreamBase stream = new ContentStreamBase.URLStream(file.toURI().toURL()); try (InputStream s = stream.getStream(); FileInputStream fis = new FileInputStream(file); GZIPInputStream zis = new GZIPInputStream(fis); diff --git a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java index 31bc572bf14..9c033827241 100644 --- a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java +++ b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java @@ -26,6 +26,8 @@ import java.lang.invoke.MethodHandles; import java.net.BindException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -782,7 +784,9 @@ public void setProxyPort(int proxyPort) { /** Returns a base URL like {@code http://localhost:8983/solr} */ public URL getBaseUrl() { try { - return new URL(protocol, host, jettyPort, "/solr"); + return new URI(protocol, null, host, jettyPort, "/solr", null, null).toURL(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } catch (MalformedURLException e) { throw new RuntimeException(e); } @@ -790,9 +794,11 @@ public URL getBaseUrl() { public URL getBaseURLV2() { try { - return new URL(protocol, host, jettyPort, "/api"); + return new URI(protocol, null, host, jettyPort, "/api", null, null).toURL(); } catch (MalformedURLException e) { throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } } @@ -802,9 +808,11 @@ public URL getBaseURLV2() { */ public URL getProxyBaseUrl() { try { - return new URL(protocol, host, getLocalPort(), "/solr"); + return new URI(protocol, null, host, getLocalPort(), "/solr", null, null).toURL(); } catch (MalformedURLException e) { throw new RuntimeException(e); + } catch (URISyntaxException e) { + throw new RuntimeException(e); } } diff --git a/solr/test-framework/src/java/org/apache/solr/handler/BackupRestoreUtils.java b/solr/test-framework/src/java/org/apache/solr/handler/BackupRestoreUtils.java index 10456e4984e..200709ef9a2 100644 --- a/solr/test-framework/src/java/org/apache/solr/handler/BackupRestoreUtils.java +++ b/solr/test-framework/src/java/org/apache/solr/handler/BackupRestoreUtils.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -109,7 +110,7 @@ public static void runReplicationHandlerCommand( } static void executeHttpRequest(String requestUrl) throws IOException { - URL url = new URL(requestUrl); + URL url = URI.create(requestUrl).toURL(); try (InputStream stream = url.openStream()) { assert stream != null; } diff --git a/solr/test-framework/src/java/org/apache/solr/handler/TestRestoreCoreUtil.java b/solr/test-framework/src/java/org/apache/solr/handler/TestRestoreCoreUtil.java index de275b8ec94..cd52cf2e68b 100644 --- a/solr/test-framework/src/java/org/apache/solr/handler/TestRestoreCoreUtil.java +++ b/solr/test-framework/src/java/org/apache/solr/handler/TestRestoreCoreUtil.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.regex.Matcher; @@ -35,7 +36,7 @@ public static boolean fetchRestoreStatus(String baseUrl, String coreName) throws + ReplicationHandler.CMD_RESTORE_STATUS; final Pattern pException = Pattern.compile("(.*?)"); - URL url = new URL(leaderUrl); + URL url = URI.create(leaderUrl).toURL(); try (InputStream stream = url.openStream()) { String response = new String(stream.readAllBytes(), StandardCharsets.UTF_8); Matcher matcher = pException.matcher(response); From b49f3d076942186d924588e602e976d13647a331 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Fri, 21 Jun 2024 10:53:20 -0700 Subject: [PATCH 007/172] SOLR-17335: New "vectorSimilarity" QParser for matching documents mased on a minimum vector similarity threshold --- solr/CHANGES.txt | 2 + .../org/apache/solr/search/QParserPlugin.java | 2 + .../neural/AbstractVectorQParserBase.java | 209 ++++ .../apache/solr/search/neural/KnnQParser.java | 183 +--- .../neural/VectorSimilarityQParser.java | 75 ++ .../neural/VectorSimilarityQParserPlugin.java | 33 + .../apache/solr/search/QueryEqualityTest.java | 74 ++ .../solr/search/neural/KnnQParserTest.java | 2 +- .../neural/VectorSimilarityQParserTest.java | 902 ++++++++++++++++++ .../pages/dense-vector-search.adoc | 103 +- 10 files changed, 1376 insertions(+), 209 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java create mode 100644 solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java create mode 100644 solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java create mode 100644 solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ca642e9bbc3..44a7af702b1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -113,6 +113,8 @@ New Features * SOLR-17277: Circuit breakers may now be configured in a "soft" or "warnOnly" mode in order to more easily test out new thresholds. Soft breakers will log out a message on each relevant request when tripped, but will not otherwise impact or short circuit the requests. (Jason Gerlowski) +* SOLR-17335: New "vectorSimilarity" QParser for matching documents mased on a minimum vector similarity threshold. (hossman) + Improvements --------------------- * SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java index 99cb9efb0c7..8146c585cf0 100644 --- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java @@ -31,6 +31,7 @@ import org.apache.solr.search.mlt.MLTContentQParserPlugin; import org.apache.solr.search.mlt.MLTQParserPlugin; import org.apache.solr.search.neural.KnnQParserPlugin; +import org.apache.solr.search.neural.VectorSimilarityQParserPlugin; import org.apache.solr.util.plugin.NamedListInitializedPlugin; public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrInfoBean { @@ -89,6 +90,7 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI map.put(HashRangeQParserPlugin.NAME, new HashRangeQParserPlugin()); map.put(RankQParserPlugin.NAME, new RankQParserPlugin()); map.put(KnnQParserPlugin.NAME, new KnnQParserPlugin()); + map.put(VectorSimilarityQParserPlugin.NAME, new VectorSimilarityQParserPlugin()); standardPlugins = Collections.unmodifiableMap(map); } diff --git a/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java b/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java new file mode 100644 index 00000000000..4cafb45744e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/neural/AbstractVectorQParserBase.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.neural; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.DenseVectorField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.QueryUtils; +import org.apache.solr.search.SyntaxError; + +public abstract class AbstractVectorQParserBase extends QParser { + + static final String PRE_FILTER = "preFilter"; + static final String EXCLUDE_TAGS = "excludeTags"; + static final String INCLUDE_TAGS = "includeTags"; + + private final String denseVectorFieldName; + private final String vectorToSearch; + + public AbstractVectorQParserBase( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + vectorToSearch = localParams.get(QueryParsing.V); + denseVectorFieldName = localParams.get(QueryParsing.F); + } + + protected String getVectorToSearch() { + if (vectorToSearch == null || vectorToSearch.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector value 'v' to search is missing"); + } + return vectorToSearch; + } + + protected String getFieldName() { + if (denseVectorFieldName == null || denseVectorFieldName.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector field 'f' is missing"); + } + return denseVectorFieldName; + } + + protected static DenseVectorField getCheckedFieldType(SchemaField schemaField) { + FieldType fieldType = schemaField.getType(); + if (!(fieldType instanceof DenseVectorField)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "only DenseVectorField is compatible with Vector Query Parsers"); + } + + return (DenseVectorField) fieldType; + } + + protected Query getFilterQuery() throws SolrException, SyntaxError { + + // Default behavior of FQ wrapping, and suitability of some local params + // depends on wether we are a sub-query or not + final boolean isSubQuery = recurseCount != 0; + + // include/exclude tags for global fqs to wrap; + // Check these up front for error handling if combined with `fq` local param. + final List includedGlobalFQTags = getLocalParamTags(INCLUDE_TAGS); + final List excludedGlobalFQTags = getLocalParamTags(EXCLUDE_TAGS); + final boolean haveGlobalFQTags = + !(includedGlobalFQTags.isEmpty() && excludedGlobalFQTags.isEmpty()); + + if (haveGlobalFQTags) { + // Some early error handling of incompatible options... + + if (isFilter()) { // this knn query is itself a filter query + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Knn Query Parser used as a filter does not support " + + INCLUDE_TAGS + + " or " + + EXCLUDE_TAGS + + " localparams"); + } + + if (isSubQuery) { // this knn query is a sub-query of a broader query (possibly disjunction) + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Knn Query Parser used as a sub-query does not support " + + INCLUDE_TAGS + + " or " + + EXCLUDE_TAGS + + " localparams"); + } + } + + // Explicit local params specifying the filter(s) to wrap + final String[] preFilters = getLocalParams().getParams(PRE_FILTER); + if (null != preFilters) { + + // We don't particularly care if preFilters is empty, the usage below will still work, + // but SolrParams API says it should be null not empty... + assert 0 != preFilters.length + : "SolrParams.getParams should return null, never zero len array"; + + if (haveGlobalFQTags) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Knn Query Parser does not support combining " + + PRE_FILTER + + " localparam with either " + + INCLUDE_TAGS + + " or " + + EXCLUDE_TAGS + + " localparams"); + } + + final List preFilterQueries = new ArrayList<>(preFilters.length); + for (String f : preFilters) { + final QParser parser = subQuery(f, null); + parser.setIsFilter(true); + + // maybe null, ie: `preFilter=""` + final Query filter = parser.getQuery(); + if (null != filter) { + preFilterQueries.add(filter); + } + } + try { + return req.getSearcher().getProcessedFilter(preFilterQueries).filter; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + + // No explicit `preFilter` localparams specifying what we should filter on. + // + // So now, if we're either a filter or a subquery, we have to default to + // not wrapping anything... + if (isFilter() || isSubQuery) { + return null; + } + + // At this point we now are a (regular) query and can wrap global `fq` filters... + try { + // Start by assuming we wrap all global filters, + // then adjust our list based on include/exclude tag params + List globalFQs = QueryUtils.parseFilterQueries(req); + + // Adjust our globalFQs based on any include/exclude we may have + if (!includedGlobalFQTags.isEmpty()) { + // NOTE: Even if no FQs match the specified tag(s) the fact that tags were specified + // means we should replace globalFQs (even with a possibly empty list) + globalFQs = new ArrayList<>(QueryUtils.getTaggedQueries(req, includedGlobalFQTags)); + } + if (null != excludedGlobalFQTags) { + globalFQs.removeAll(QueryUtils.getTaggedQueries(req, excludedGlobalFQTags)); + } + + return req.getSearcher().getProcessedFilter(globalFQs).filter; + + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + + /** + * @return set (possibly empty) of tags specified in the given local param + * @see StrUtils#splitSmart + * @see QueryUtils#getTaggedQueries + * @see #localParams + */ + private List getLocalParamTags(final String param) { + final String[] strVals = localParams.getParams(param); + if (null == strVals) { + return Collections.emptyList(); + } + final List tags = new ArrayList<>(strVals.length * 2); + for (String val : strVals) { + // This ensures parity w/how QParser constructor builds tagMap, + // and that empty strings will make it into our List (for "include nothing") + if (0 < val.indexOf(',')) { + tags.addAll(StrUtils.splitSmart(val, ',')); + } else { + tags.add(val); + } + } + return tags; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java index 252a4fcabc7..166dada5b7f 100644 --- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java +++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java @@ -16,204 +16,31 @@ */ package org.apache.solr.search.neural; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import org.apache.lucene.search.Query; -import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.DenseVectorField; -import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; -import org.apache.solr.search.QParser; -import org.apache.solr.search.QueryParsing; -import org.apache.solr.search.QueryUtils; import org.apache.solr.search.SyntaxError; -public class KnnQParser extends QParser { - - static final String PRE_FILTER = "preFilter"; - static final String EXCLUDE_TAGS = "excludeTags"; - static final String INCLUDE_TAGS = "includeTags"; +public class KnnQParser extends AbstractVectorQParserBase { // retrieve the top K results based on the distance similarity function static final String TOP_K = "topK"; static final int DEFAULT_TOP_K = 10; - /** - * Constructor for the QParser - * - * @param qstr The part of the query string specific to this parser - * @param localParams The set of parameters that are specific to this QParser. See - * https://solr.apache.org/guide/solr/latest/query-guide/local-params.html - * @param params The rest of the {@link SolrParams} - * @param req The original {@link SolrQueryRequest}. - */ public KnnQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { super(qstr, localParams, params, req); } @Override public Query parse() throws SyntaxError { - String denseVectorField = localParams.get(QueryParsing.F); - String vectorToSearch = localParams.get(QueryParsing.V); - int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K); - - if (denseVectorField == null || denseVectorField.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector field 'f' is missing"); - } - - if (vectorToSearch == null || vectorToSearch.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "the Dense Vector value 'v' to search is missing"); - } - - SchemaField schemaField = req.getCore().getLatestSchema().getField(denseVectorField); - FieldType fieldType = schemaField.getType(); - if (!(fieldType instanceof DenseVectorField)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "only DenseVectorField is compatible with Knn Query Parser"); - } - - DenseVectorField denseVectorType = (DenseVectorField) fieldType; + final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName()); + final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); + final String vectorToSearch = getVectorToSearch(); + final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K); return denseVectorType.getKnnVectorQuery( schemaField.getName(), vectorToSearch, topK, getFilterQuery()); } - - private Query getFilterQuery() throws SolrException, SyntaxError { - - // Default behavior of FQ wrapping, and suitability of some local params - // depends on wether we are a sub-query or not - final boolean isSubQuery = recurseCount != 0; - - // include/exclude tags for global fqs to wrap; - // Check these up front for error handling if combined with `fq` local param. - final List includedGlobalFQTags = getLocalParamTags(INCLUDE_TAGS); - final List excludedGlobalFQTags = getLocalParamTags(EXCLUDE_TAGS); - final boolean haveGlobalFQTags = - !(includedGlobalFQTags.isEmpty() && excludedGlobalFQTags.isEmpty()); - - if (haveGlobalFQTags) { - // Some early error handling of incompatible options... - - if (isFilter()) { // this knn query is itself a filter query - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Knn Query Parser used as a filter does not support " - + INCLUDE_TAGS - + " or " - + EXCLUDE_TAGS - + " localparams"); - } - - if (isSubQuery) { // this knn query is a sub-query of a broader query (possibly disjunction) - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Knn Query Parser used as a sub-query does not support " - + INCLUDE_TAGS - + " or " - + EXCLUDE_TAGS - + " localparams"); - } - } - - // Explicit local params specifying the filter(s) to wrap - final String[] preFilters = getLocalParams().getParams(PRE_FILTER); - if (null != preFilters) { - - // We don't particularly care if preFilters is empty, the usage below will still work, - // but SolrParams API says it should be null not empty... - assert 0 != preFilters.length - : "SolrParams.getParams should return null, never zero len array"; - - if (haveGlobalFQTags) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Knn Query Parser does not support combining " - + PRE_FILTER - + " localparam with either " - + INCLUDE_TAGS - + " or " - + EXCLUDE_TAGS - + " localparams"); - } - - final List preFilterQueries = new ArrayList<>(preFilters.length); - for (String f : preFilters) { - final QParser parser = subQuery(f, null); - parser.setIsFilter(true); - - // maybe null, ie: `preFilter=""` - final Query filter = parser.getQuery(); - if (null != filter) { - preFilterQueries.add(filter); - } - } - try { - return req.getSearcher().getProcessedFilter(preFilterQueries).filter; - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - } - - // No explicit `preFilter` localparams specifying what we should filter on. - // - // So now, if we're either a filter or a subquery, we have to default to - // not wrapping anything... - if (isFilter() || isSubQuery) { - return null; - } - - // At this point we now are a (regular) query and can wrap global `fq` filters... - try { - // Start by assuming we wrap all global filters, - // then adjust our list based on include/exclude tag params - List globalFQs = QueryUtils.parseFilterQueries(req); - - // Adjust our globalFQs based on any include/exclude we may have - if (!includedGlobalFQTags.isEmpty()) { - // NOTE: Even if no FQs match the specified tag(s) the fact that tags were specified - // means we should replace globalFQs (even with a possibly empty list) - globalFQs = new ArrayList<>(QueryUtils.getTaggedQueries(req, includedGlobalFQTags)); - } - if (null != excludedGlobalFQTags) { - globalFQs.removeAll(QueryUtils.getTaggedQueries(req, excludedGlobalFQTags)); - } - - return req.getSearcher().getProcessedFilter(globalFQs).filter; - - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - } - - /** - * @return set (possibly empty) of tags specified in the given local param - * @see StrUtils#splitSmart - * @see QueryUtils#getTaggedQueries - * @see #localParams - */ - private List getLocalParamTags(final String param) { - final String[] strVals = localParams.getParams(param); - if (null == strVals) { - return Collections.emptyList(); - } - final List tags = new ArrayList<>(strVals.length * 2); - for (String val : strVals) { - // This ensures parity w/how QParser constructor builds tagMap, - // and that empty strings will make it into our List (for "include nothing") - if (0 < val.indexOf(',')) { - tags.addAll(StrUtils.splitSmart(val, ',')); - } else { - tags.add(val); - } - } - return tags; - } } diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java new file mode 100644 index 00000000000..e3ec2f242f7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParser.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.neural; + +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.search.ByteVectorSimilarityQuery; +import org.apache.lucene.search.FloatVectorSimilarityQuery; +import org.apache.lucene.search.Query; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.DenseVectorField; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.vector.DenseVectorParser; + +public class VectorSimilarityQParser extends AbstractVectorQParserBase { + + // retrieve the top results based on the distance similarity function thresholds + static final String MIN_RETURN = "minReturn"; + static final String MIN_TRAVERSE = "minTraverse"; + + static final float DEFAULT_MIN_TRAVERSE = Float.NEGATIVE_INFINITY; + + public VectorSimilarityQParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + final String fieldName = getFieldName(); + final SchemaField schemaField = req.getCore().getLatestSchema().getField(fieldName); + final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); + final String vectorToSearch = getVectorToSearch(); + final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE); + final Float minReturn = localParams.getFloat(MIN_RETURN); + if (null == minReturn) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + MIN_RETURN + " is required to use Vector Similarity QParser"); + } + + final DenseVectorParser vectorBuilder = + denseVectorType.getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY); + + final VectorEncoding vectorEncoding = denseVectorType.getVectorEncoding(); + switch (vectorEncoding) { + case FLOAT32: + return new FloatVectorSimilarityQuery( + fieldName, vectorBuilder.getFloatVector(), minTraverse, minReturn, getFilterQuery()); + case BYTE: + return new ByteVectorSimilarityQuery( + fieldName, vectorBuilder.getByteVector(), minTraverse, minReturn, getFilterQuery()); + default: + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Unexpected state. Vector Encoding: " + vectorEncoding); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java new file mode 100644 index 00000000000..4109d0df388 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/neural/VectorSimilarityQParserPlugin.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.neural; + +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; + +/** A neural query parser to run min-similarity search on Dense Vector fields. */ +public class VectorSimilarityQParserPlugin extends QParserPlugin { + public static final String NAME = "vectorSimilarity"; + + @Override + public QParser createParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new VectorSimilarityQParser(qstr, localParams, params, req); + } +} diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java index 653c0935879..6e675771560 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -1469,6 +1469,80 @@ public void testQueryKNN() throws Exception { } } + public void testQueryVecSim() throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "0"); + doc.addField("vector", Arrays.asList(1, 2, 3, 4)); + assertU(adoc(doc)); + assertU(commit()); + + final String common = "!vectorSimilarity minReturn=0.3 f=vector"; + final String qvec = "[1.0,2.0,3.0,4.0]"; + + try (SolrQueryRequest req0 = req()) { + + // no filters + final Query fqNull = + assertQueryEqualsAndReturn( + "vectorSimilarity", + req0, + "{" + common + "}" + qvec, + "{" + common + " minTraverse='-Infinity'}" + qvec, + "{" + common + " preFilter=''}" + qvec, + "{" + common + " v=" + qvec + "}"); + + try (SolrQueryRequest req1 = req("fq", "{!tag=t1}id:1", "xxx", "id:1")) { + // either global fq, or (same) preFilter as localparam + final Query fqOne = + assertQueryEqualsAndReturn( + "vectorSimilarity", + req1, + "{" + common + "}" + qvec, + "{" + common + " includeTags=t1}" + qvec, + "{" + common + " preFilter='id:1'}" + qvec, + "{" + common + " preFilter=$xxx}" + qvec, + "{" + common + " v=" + qvec + "}"); + QueryUtils.checkUnequal(fqNull, fqOne); + + try (SolrQueryRequest req2 = req("fq", "{!tag=t2}id:2", "xxx", "id:1", "yyy", "")) { + // override global fq with local param to use different preFilter + final Query fqOneOverride = + assertQueryEqualsAndReturn( + "vectorSimilarity", + req2, + "{" + common + " preFilter='id:1'}" + qvec, + "{" + common + " preFilter=$xxx}" + qvec); + QueryUtils.checkEqual(fqOne, fqOneOverride); + + // override global fq with local param to use no preFilters + final Query fqNullOverride = + assertQueryEqualsAndReturn( + "vectorSimilarity", + req2, + "{" + common + " preFilter=''}" + qvec, + "{" + common + " excludeTags=t2}" + qvec, + "{" + common + " preFilter=$yyy}" + qvec); + QueryUtils.checkEqual(fqNull, fqNullOverride); + } + } + + try (SolrQueryRequest reqPostFilter = req("fq", "{!tag=post frange cache=false l=0}9.9")) { + // global post-filter fq should always be ignored + final Query fqPostFilter = + assertQueryEqualsAndReturn( + "vectorSimilarity", + reqPostFilter, + "{" + common + "}" + qvec, + "{" + common + " includeTags=post}" + qvec); + QueryUtils.checkEqual(fqNull, fqPostFilter); + } + + } finally { + delQ("id:0"); + assertU(commit()); + } + } + /** * NOTE: defType is not only used to pick the parser, but also to record the parser being tested * for coverage sanity checking diff --git a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java index ccd0e3ecc44..f5d5668a7e5 100644 --- a/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java +++ b/solr/core/src/test/org/apache/solr/search/neural/KnnQParserTest.java @@ -178,7 +178,7 @@ public void incorrectVectorFieldType_shouldThrowException() { assertQEx( "Incorrect vector field type should throw Exception", - "only DenseVectorField is compatible with Knn Query Parser", + "only DenseVectorField is compatible with Vector Query Parsers", req(CommonParams.Q, "{!knn f=id topK=10}" + vectorToSearch, "fl", "id"), SolrException.ErrorCode.BAD_REQUEST); } diff --git a/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java b/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java new file mode 100644 index 00000000000..cf6aeb65973 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/neural/VectorSimilarityQParserTest.java @@ -0,0 +1,902 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.neural; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class VectorSimilarityQParserTest extends SolrTestCaseJ4 { + String IDField = "id"; + String vectorField = "vector"; + String vectorField2 = "vector2"; + String vectorFieldByteEncoding = "vector_byte_encoding"; + + @Before + public void prepareIndex() throws Exception { + /* vectorDimension="4" similarityFunction="cosine" */ + initCore("solrconfig_codec.xml", "schema-densevector.xml"); + + List docsToIndex = this.prepareDocs(); + for (SolrInputDocument doc : docsToIndex) { + assertU(adoc(doc)); + } + + assertU(commit()); + } + + private List prepareDocs() { + int docsCount = 13; + List docs = new ArrayList<>(docsCount); + for (int i = 1; i < docsCount + 1; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(IDField, i); + docs.add(doc); + } + + docs.get(0) + .addField(vectorField, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector1= 1.0 + docs.get(1) + .addField( + vectorField, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector1= 0.998 + docs.get(2) + .addField( + vectorField, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector1= 0.992 + docs.get(3) + .addField( + vectorField, Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f)); // cosine distance vector1= 0.999 + docs.get(4) + .addField(vectorField, Arrays.asList(30f, 22f, 35f, 20f)); // cosine distance vector1= 0.862 + docs.get(5) + .addField(vectorField, Arrays.asList(40f, 1f, 1f, 200f)); // cosine distance vector1= 0.756 + docs.get(6) + .addField(vectorField, Arrays.asList(5f, 10f, 20f, 40f)); // cosine distance vector1= 0.970 + docs.get(7) + .addField( + vectorField, Arrays.asList(120f, 60f, 30f, 15f)); // cosine distance vector1= 0.515 + docs.get(8) + .addField( + vectorField, Arrays.asList(200f, 50f, 100f, 25f)); // cosine distance vector1= 0.554 + docs.get(9) + .addField( + vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997 + docs.get(10) + .addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1 + docs.get(11) + .addField( + vectorField2, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector2= 0.992 + docs.get(12) + .addField( + vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998 + + docs.get(0).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 3, 4)); + docs.get(1).addField(vectorFieldByteEncoding, Arrays.asList(2, 2, 1, 4)); + docs.get(2).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 1, 2)); + docs.get(3).addField(vectorFieldByteEncoding, Arrays.asList(7, 2, 1, 3)); + docs.get(4).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(5).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(6).addField(vectorFieldByteEncoding, Arrays.asList(18, 2, 4, 4)); + docs.get(7).addField(vectorFieldByteEncoding, Arrays.asList(8, 3, 2, 4)); + + return docs; + } + + @After + public void cleanUp() { + clearIndex(); + deleteCore(); + } + + @Test + public void incorrectVectorFieldType_shouldThrowException() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQEx( + "Incorrect vector field type should throw Exception", + "only DenseVectorField is compatible with ", + req(CommonParams.Q, "{!vectorSimilarity f=id}" + vectorToSearch, "fl", "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void undefinedVectorField_shouldThrowException() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQEx( + "Undefined vector field should throw Exception", + "undefined field: \"notExistent\"", + req(CommonParams.Q, "{!vectorSimilarity f=notExistent}" + vectorToSearch, "fl", "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void missingVectorField_shouldThrowException() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQEx( + "missing vector field should throw Exception", + "the Dense Vector field 'f' is missing", + req(CommonParams.Q, "{!vectorSimilarity}" + vectorToSearch, "fl", "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void correctVectorField_shouldSearchOnThatField() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQ( + req( + CommonParams.Q, + "{!vectorSimilarity f=vector2 minReturn=0.8}" + vectorToSearch, + "fl", + "id"), + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='11']", + "//result/doc[2]/str[@name='id'][.='13']", + "//result/doc[3]/str[@name='id'][.='12']"); + } + + @Test + public void vectorByteEncodingField_shouldSearchOnThatField() { + String vectorToSearch = "[2, 2, 1, 3]"; + + float minR = + VectorSimilarityFunction.COSINE.compare(new byte[] {2, 2, 1, 3}, new byte[] {1, 2, 1, 2}); + assertQ( + req( + CommonParams.Q, + "{!vectorSimilarity f=vector_byte_encoding minReturn=" + minR + "}" + vectorToSearch, + "fl", + "id"), + "//result[@numFound='2']", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='3']"); + + vectorToSearch = "[8, 3, 2, 4]"; + minR = + VectorSimilarityFunction.COSINE.compare(new byte[] {8, 3, 2, 4}, new byte[] {7, 2, 1, 3}); + + assertQ( + req( + CommonParams.Q, + "{!vectorSimilarity f=vector_byte_encoding minReturn=" + minR + "}" + vectorToSearch, + "fl", + "id,score,vector_byte_encoding", + "indent", + "true"), + "//result[@numFound='2']", + "//result/doc[1]/str[@name='id'][.='8']", + "//result/doc[2]/str[@name='id'][.='4']"); + } + + @Test + public void vectorByteEncodingField_shouldRaiseExceptionIfQueryUsesFloatVectors() { + String vectorToSearch = "[8.3, 4.3, 2.1, 4.1]"; + + assertQEx( + "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + "incorrect vector element: '8.3'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void + vectorByteEncodingField_shouldRaiseExceptionWhenQueryContainsValuesOutsideByteValueRange() { + String vectorToSearch = "[1, -129, 3, 5]"; + + assertQEx( + "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + "incorrect vector element: ' -129'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "[1, 3, 156, 5]"; + + assertQEx( + "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + "incorrect vector element: ' 156'. The expected format is:'[b1,b2..b3]' where each element b is a byte (-128 to 127)", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector_byte_encoding minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void missingVectorToSearch_shouldThrowException() { + assertQEx( + "missing vector to search should throw Exception", + "the Dense Vector value 'v' to search is missing", + req(CommonParams.Q, "{!vectorSimilarity f=vector minReturn=0.0}", "fl", "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void incorrectVectorToSearchDimension_shouldThrowException() { + String vectorToSearch = "[2.0, 4.4, 3.]"; + assertQEx( + "missing vector to search should throw Exception", + "incorrect vector dimension. The vector value has size 3 while it is expected a vector with size 4", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "[2.0, 4.4,,]"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector dimension. The vector value has size 2 while it is expected a vector with size 4", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void incorrectVectorToSearch_shouldThrowException() { + String vectorToSearch = "2.0, 4.4, 3.5, 6.4"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "[2.0, 4.4, 3.5, 6.4"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "2.0, 4.4, 3.5, 6.4]"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector format. The expected format is:'[f1,f2..f3]' where each element f is a float", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "[2.0, 4.4, 3.5, stringElement]"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector element: ' stringElement'. The expected format is:'[f1,f2..f3]' where each element f is a float", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + + vectorToSearch = "[2.0, 4.4, , ]"; + assertQEx( + "incorrect vector to search should throw Exception", + "incorrect vector element: ' '. The expected format is:'[f1,f2..f3]' where each element f is a float", + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.0}" + vectorToSearch, + "fl", + "id"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void correctQuery_shouldRankBySimilarityFunction() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQ( + req( + CommonParams.Q, + "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch, + "fl", + "id"), + "//result[@numFound='8']", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='2']", + "//result/doc[4]/str[@name='id'][.='10']", + "//result/doc[5]/str[@name='id'][.='3']", + "//result/doc[6]/str[@name='id'][.='7']", + "//result/doc[7]/str[@name='id'][.='5']", + "//result/doc[8]/str[@name='id'][.='6']"); + } + + @Test + public void vecSimQueryUsedInFilters_shouldFilterResultsBeforeTheQueryExecution() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + // minReturn=0.8 -> 1,4,2,10,3,7,5,6 + assertQ( + req( + CommonParams.Q, + "id:(3 4 9 2)", + "fq", + "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch, + "fq", + "id:(4 20 9)", + "fl", + "id"), + "//result[@numFound='1']", + "//result/doc[1]/str[@name='id'][.='4']"); + } + + @Test + public void vecSimQueryUsedInFiltersWithPreFilter_shouldFilterResultsBeforeTheQueryExecution() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + // minReturn=0.8 w/localparam preFilter -> 1,4,7 + assertQ( + req( + CommonParams.Q, + "id:(3 4 7 2)", + "fq", + "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:(1 4 7 8 9)'}" + + vectorToSearch, + "fq", + "id:(4 20 7)", + "fl", + "id"), + "//result[@numFound='2']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='7']"); + } + + @Test + public void vecSimQueryUsedInFilters_rejectIncludeExclude() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + for (String fq : + Arrays.asList( + "{!vectorSimilarity f=vector minReturn=0.8 includeTags=xxx}" + vectorToSearch, + "{!vectorSimilarity f=vector minReturn=0.8 excludeTags=xxx}" + vectorToSearch)) { + assertQEx( + "fq={!vectorSimilarity...} incompatible with include/exclude localparams", + "used as a filter does not support", + req("q", "*:*", "fq", fq), + SolrException.ErrorCode.BAD_REQUEST); + } + } + + @Test + public void vecSimQueryAsSubQuery() { + final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]"); + final String filt = "id:(2 4 7 9 8 20)"; + + // When vecSim parser is a subquery, it should not pre-filter on any global fq params + // minReturn -> 1,4,2,10,3,7,5,6 -> fq -> 4,2,7 + assertQ( + req(common, "fq", filt, "q", "*:* AND {!vectorSimilarity f=vector minReturn=0.8 v=$vec}"), + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='7']"); + // minReturn -> 1,4,2,10,3,7,5,6 + '8' -> fq -> 4,2,7,8 + assertQ( + req( + common, + "fq", + filt, + "q", + "id:8^=0.01 OR {!vectorSimilarity f=vector minReturn=0.8 v=$vec}"), + "//result[@numFound='4']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='7']", + "//result/doc[4]/str[@name='id'][.='8']"); + } + + @Test + public void vecSimQueryAsSubQuery_withPreFilter() { + final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]"); + final String filt = "id:(2 4 7 9 8 20 3)"; + + // vecSim subquery should still accept `preFilter` local param + // filt -> minReturn -> 4,2,3,7 + assertQ( + req( + common, + "q", + "*:* AND {!vectorSimilarity f=vector minReturn=0.8 preFilter='" + filt + "' v=$vec}"), + "//result[@numFound='4']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='7']"); + + // it should not pre-filter on any global fq params + // filt -> minReturn -> 4,2,3,7 -> fq -> 3,7 + assertQ( + req( + common, + "fq", + "id:(1 9 20 3 5 7 8)", + "q", + "*:* AND {!vectorSimilarity f=vector minReturn=0.8 preFilter='" + filt + "' v=$vec}"), + "//result[@numFound='2']", + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='7']"); + // filt -> minReturn -> 4,2,3,7 + '8' -> fq -> 8,3,7 + assertQ( + req( + common, + "fq", + "id:(1 9 20 3 5 7 8)", + "q", + "id:8^=100 OR {!vectorSimilarity f=vector minReturn=0.8 preFilter='" + + filt + + "' v=$vec}"), + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='8']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='7']"); + } + + @Test + public void vecSimQueryAsSubQuery_rejectIncludeExclude() { + final SolrParams common = params("fl", "id", "vec", "[1.0, 2.0, 3.0, 4.0]"); + + for (String subq : + Arrays.asList( + "{!vectorSimilarity f=vector minReturn=0.8 includeTags=xxx v=$vec}", + "{!vectorSimilarity f=vector minReturn=0.8 excludeTags=xxx v=$vec}")) { + assertQEx( + "vecSim as subquery incompatible with include/exclude localparams", + "used as a sub-query does not support", + req(common, "q", "*:* OR " + subq), + SolrException.ErrorCode.BAD_REQUEST); + } + } + + @Test + public void vecSimQueryWithFilterQuery_singlePreFilterEquivilence() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + final SolrParams common = params("fl", "id"); + + // these requests should be equivalent + final String filt = "id:(1 2 7 20)"; + for (SolrQueryRequest req : + Arrays.asList( + req( + common, + "q", + "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch, + "fq", + filt), + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter=\"" + + filt + + "\" minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter=$my_filt minReturn=0.8}" + vectorToSearch, + "my_filt", + filt))) { + assertQ( + req, + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='7']"); + } + } + + @Test + public void vecSimQueryWithFilterQuery_multiPreFilterEquivilence() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + final SolrParams common = params("fl", "id"); + + // these requests should be equivalent + final String fx = "id:(3 4 9 2 1 )"; // 1 & 10 dropped from intersection + final String fy = "id:(3 4 9 2 10)"; + final String minR = "minReturn=0.8"; // should exclude 9 + for (SolrQueryRequest req : + Arrays.asList( + req( + common, + "q", + "{!vectorSimilarity f=vector " + minR + "}" + vectorToSearch, + "fq", + fx, + "fq", + fy), + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter=\"" + + fx + + "\" preFilter=\"" + + fy + + "\" " + + minR + + "}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter=$fx preFilter=$fy " + + minR + + "}" + + vectorToSearch, + "fx", + fx, + "fy", + fy), + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter=$multi_filt " + minR + "}" + vectorToSearch, + "multi_filt", + fx, + "multi_filt", + fy))) { + assertQ( + req, + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']"); + } + } + + @Test + public void vecSimQueryWithPreFilter_rejectIncludeExclude() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQEx( + "vecSim preFilter localparm incompatible with include/exclude localparams", + "does not support combining preFilter localparam with either", + // shouldn't matter if global fq w/tag even exists, usage is an error + req( + "q", + "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:1' includeTags=xxx}" + + vectorToSearch), + SolrException.ErrorCode.BAD_REQUEST); + assertQEx( + "vecSim preFilter localparm incompatible with include/exclude localparams", + "does not support combining preFilter localparam with either", + // shouldn't matter if global fq w/tag even exists, usage is an error + req( + "q", + "{!vectorSimilarity f=vector minReturn=0.8 preFilter='id:1' excludeTags=xxx}" + + vectorToSearch), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void vecSimQueryWithFilterQuery_preFilterLocalParamOverridesGlobalFilters() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + // trivial case: empty preFilter localparam means no pre-filtering + assertQ( + req( + "q", "{!vectorSimilarity f=vector preFilter='' minReturn=0.8}" + vectorToSearch, + "fq", "-id:4", + "fl", "id"), + "//result[@numFound='7']", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='10']", + "//result/doc[4]/str[@name='id'][.='3']"); + + // localparam prefiltering, global fqs applied independently + assertQ( + req( + "q", + "{!vectorSimilarity f=vector preFilter='id:(3 4 9 2 7 8)' minReturn=0.8}" + + vectorToSearch, + "fq", "-id:4", + "fl", "id"), + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='7']"); + } + + @Test + public void vecSimQueryWithFilterQuery_localParamIncludeExcludeTags() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + final SolrParams common = + params( + "fl", "id", + "fq", "{!tag=xx,aa}id:(5 6 7 8 9 10)", + "fq", "{!tag=yy,aa}id:(1 2 3 4 5 6 7)"); + + // These req's are equivalent: pre-filter everything + // So only 7,6,5 are viable for minReturn=0.8 + for (SolrQueryRequest req : + Arrays.asList( + // default behavior is all fq's pre-filter, + req(common, "q", "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch), + // diff ways of explicitly requesting both fq params + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa excludeTags='' minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa excludeTags=bogus minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=xx includeTags=yy minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=xx,yy,bogus minReturn=0.8}" + + vectorToSearch))) { + assertQ( + req, + "//result[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='7']", + "//result/doc[2]/str[@name='id'][.='5']", + "//result/doc[3]/str[@name='id'][.='6']"); + } + } + + @Test + public void vecSimQueryWithFilterQuery_localParamsDisablesAllPreFiltering() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + final SolrParams common = + params( + "fl", "id", + "fq", "{!tag=xx,aa}id:(11 7 8 9 10)", + "fq", "{!tag=yy,aa}id:(1 2 3 4 12 7)"); + + // These req's are equivalent: pre-filter nothing + // So 1,4,2,10,3,7,5,6 are the minReturn=0.8 + // Only 7 matches both of the the regular fq params + for (SolrQueryRequest req : + Arrays.asList( + // explicit local empty preFilter + req( + common, + "q", + "{!vectorSimilarity f=vector preFilter='' minReturn=0.8}" + vectorToSearch), + // diff ways of explicitly including none of the global fq params + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags='' minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=bogus minReturn=0.8}" + vectorToSearch), + // diff ways of explicitly excluding all of the global fq params + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=aa minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa excludeTags=aa minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa excludeTags=xx,yy minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=xx,yy excludeTags=aa minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=xx,yy minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=aa minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=xx excludeTags=yy minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=xx excludeTags=yy,bogus minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=xx,yy,bogus minReturn=0.8}" + + vectorToSearch))) { + assertQ(req, "//result[@numFound='1']", "//result/doc[1]/str[@name='id'][.='7']"); + } + } + + @Test + public void vecSimQueryWithFilterQuery_localParamCombinedIncludeExcludeTags() { + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + final SolrParams common = + params( + "fl", "id", + "fq", "{!tag=xx,aa}id:(11 7 8 9 10)", + "fq", "{!tag=yy,aa}id:(1 2 3 4 12 7)"); + + // These req's are equivalent: prefilter only the 'yy' fq + // So 1,4,2,3,7 are in the minReturn=0.8 + // Only 7 matches the regular 'xx' fq param + for (SolrQueryRequest req : + Arrays.asList( + // diff ways of only using the 'yy' filter + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=yy,bogus minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=yy excludeTags='' minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector excludeTags=xx,bogus minReturn=0.8}" + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=yy excludeTags=xx minReturn=0.8}" + + vectorToSearch), + req( + common, + "q", + "{!vectorSimilarity f=vector includeTags=aa excludeTags=xx minReturn=0.8}" + + vectorToSearch))) { + assertQ(req, "//result[@numFound='1']", "//result/doc[1]/str[@name='id'][.='7']"); + } + } + + @Test + public void vecSimQueryWithMultiSelectFaceting_excludeTags() { + // NOTE: faceting on id is not very realistic, + // but it confirms what we care about re:filters w/o needing extra fields. + final String facet_xpath = "//lst[@name='facet_fields']/lst[@name='id']/int"; + final String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + final SolrParams common = + params( + "fl", "id", + "indent", "true", + "q", "{!vectorSimilarity f=vector minReturn=0.9 excludeTags=facet_click v=$vec}", + "vec", vectorToSearch, + // mimicing "inStock:true" + "fq", "-id:(2 3)", + "facet", "true", + "facet.mincount", "1", + "facet.field", "{!ex=facet_click}id"); + + // initial query, with basic pre-filter and facet counts + assertQ( + req(common), + "//result[@numFound='5']", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='4']", + "//result/doc[3]/str[@name='id'][.='10']", + "//result/doc[4]/str[@name='id'][.='7']", + "//result/doc[5]/str[@name='id'][.='5']", + "*[count(" + facet_xpath + ")=5]", + facet_xpath + "[@name='1'][.='1']", + facet_xpath + "[@name='4'][.='1']", + facet_xpath + "[@name='10'][.='1']", + facet_xpath + "[@name='7'][.='1']", + facet_xpath + "[@name='5'][.='1']"); + + // drill down on a single facet constraint + // multi-select means facet counts shouldn't change + assertQ( + req(common, "fq", "{!tag=facet_click}id:(4)"), + "//result[@numFound='1']", + "//result/doc[1]/str[@name='id'][.='4']", + "*[count(" + facet_xpath + ")=5]", + facet_xpath + "[@name='1'][.='1']", + facet_xpath + "[@name='4'][.='1']", + facet_xpath + "[@name='10'][.='1']", + facet_xpath + "[@name='7'][.='1']", + facet_xpath + "[@name='5'][.='1']"); + + // drill down on an additional facet constraint + // multi-select means facet counts shouldn't change + assertQ( + req(common, "fq", "{!tag=facet_click}id:(4 5)"), + "//result[@numFound='2']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='5']", + "*[count(" + facet_xpath + ")=5]", + facet_xpath + "[@name='1'][.='1']", + facet_xpath + "[@name='4'][.='1']", + facet_xpath + "[@name='10'][.='1']", + facet_xpath + "[@name='7'][.='1']", + facet_xpath + "[@name='5'][.='1']"); + } + + /** + * See {@link org.apache.solr.search.ReRankQParserPlugin.ReRankQueryRescorer#combine(float, + * boolean, float)}} for more details. + */ + @Test + public void vecSimQueryAsRerank_shouldAddSimilarityFunctionScore() { + String vectorToSearch = "[1.0, 2.0, 3.0, 4.0]"; + + assertQ( + req( + CommonParams.Q, + "id:(3 4 9 2)", + "rq", + "{!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}", + "rqq", + "{!vectorSimilarity f=vector minReturn=0.8}" + vectorToSearch, + "fl", + "id"), + "//result[@numFound='4']", + "//result/doc[1]/str[@name='id'][.='4']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='3']", + "//result/doc[4]/str[@name='id'][.='9']"); + } +} diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index ac96ef827bf..35f5f094c68 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -237,14 +237,12 @@ client.add(Arrays.asList(d1, d2)); -- == Query Time -This is the Apache Solr query approach designed to support dense vector search: -=== knn Query Parser -The `knn` k-nearest neighbors query parser allows to find the k-nearest documents to the target vector according to indexed dense vectors in the given field. The set of documents can be Pre-Filtered to reduce the number of vector distance calculations that must be computed, and ensure the best `topK` are returned. +Apache Solr provides two query parsers that work with dense vector fields, that each support different ways of matching documents based on vector similarity: The `knn` query parser, and the `vectorSimilarity` query parser. -The score for a retrieved document is the approximate distance to the target vector(defined by the similarityFunction configured at indexing time). +Both parsers return scores for retrieved documents that are the approximate distance to the target vector (defined by the similarityFunction configured at indexing time) and both support "Pre-Filtering" the document graph to reduce the number of candidate vectors evaluated (with out needing to compute their vector similarity distances). -It takes the following parameters: +Common parameters for both query parsers are: `f`:: + @@ -255,15 +253,6 @@ s|Required |Default: none + The `DenseVectorField` to search in. -`topK`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: 10 -|=== -+ -How many k-nearest results to return. - `preFilter`:: + [%autowidth,frame=none] @@ -293,22 +282,73 @@ Indicates that only `fq` filters with the specified `tag` should be considered f Indicates that `fq` filters with the specified `tag` should be excluded from consideration for implicit Pre-Filtering. Must not be combined with `preFilter`. -Here's how to run a simple KNN search: +=== knn Query Parser + +The `knn` k-nearest neighbors query parser matches k-nearest documents to the target vector. + +In addition to the common parameters described above, it takes the following parameters: + +`topK`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: 10 +|=== ++ +How many k-nearest results to return. + +Here's an example of a simple `knn` search: [source,text] ?q={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0] The search results retrieved are the k=10 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, ranked by the `similarityFunction` configured at indexing time. +=== vectorSimilarity Query Parser + +The `vectorSimilarity` vector similarity query parser matches documents whose similarity with the target vector is a above a minimum threshold. -==== Explicit KNN Pre-Filtering +In addition to the common parameters described above, it takes the following parameters: -The `knn` query parser's `preFilter` parameter can be specified to reduce the number of candidate documents evaluated for the k-nearest distance calculation: + +`minReturn`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +Minimum similarity threshold of nodes in the graph to be returned as matches + +`minTraverse`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: -Infinity +|=== ++ +Minimum similarity of nodes in the graph to continue traversal of their neighbors + +Here's an example of a simple `vectorSimilarity` search: + +[source,text] +?q={!vectorSimilarity f=vector minReturn=0.7}[1.0, 2.0, 3.0, 4.0] + +The search results retrieved are all documents whose similarity with the input vector `[1.0, 2.0, 3.0, 4.0]` is at least `0.7` based on the `similarityFunction` configured at indexing time + + +=== Graph Pre-Filtering + +Pre-Filtering the set of candidate documents considered when walking the graph can be specified either explicitly, or implicitly (based on existing `fq` params) depending on how and when these dense vector query parsers are used. + +==== Explicit Pre-Filtering + +The `preFilter` parameter can be specified explicitly to reduce the number of candidate documents evaluated for the distance calculation: [source,text] -?q={!knn f=vector topK=10 preFilter=inStock:true}[1.0, 2.0, 3.0, 4.0] +?q={!vectorSimilarity f=vector minReturn=0.7 preFilter=inStock:true}[1.0, 2.0, 3.0, 4.0] -In the above example, only documents matching the Pre-Filter `inStock:true` will be candidates for consideration when evaluating the k-nearest search against the specified vector. +In the above example, only documents matching the Pre-Filter `inStock:true` will be candidates for consideration when evaluating the `vectorSimilarity` search against the specified vector. The `preFilter` parameter may be blank (ex: `preFilter=""`) to indicate that no Pre-Filtering should be performed; or it may be multi-valued -- either through repetition, or via duplicated xref:local-params.adoc#parameter-dereferencing[Parameter References]. @@ -324,22 +364,22 @@ These two examples are equivalent: &knnPreFilter=inStock:true ---- -==== Implicit KNN Pre-Filtering +==== Implicit Pre-Filtering -While the `preFilter` parameter may be explicitly specified on *_any_* usage of the `knn` query parser, the default Pre-Filtering behavior (when no `preFilter` parameter is specified) will vary based on how the `knn` query parser is used: +While the `preFilter` parameter may be explicitly specified on *_any_* usage of the `knn` or `vectorSimilarity` query parsers, the default Pre-Filtering behavior (when no `preFilter` parameter is specified) will vary based on how the query parser is used: -* When used as the main `q` param: `fq` filters in the request (that are not xref:common-query-parameters.adoc#cache-local-parameter[Solr Post Filters]) will be combined to form an implicit KNN Pre-Filter. +* When used as the main `q` param: `fq` filters in the request (that are not xref:common-query-parameters.adoc#cache-local-parameter[Solr Post Filters]) will be combined to form an implicit Graph Pre-Filter. ** This default behavior optimizes the number of vector distance calculations considered, eliminating documents that would eventually be excluded by an `fq` filter anyway. ** `includeTags` and `excludeTags` may be used to limit the set of `fq` filters used in the Pre-Filter. -* When used as an `fq` param, or as a subquery clause in a larger query: No implicit Pre-Filter is used. +* When a vector search query parser is used as an `fq` param, or as a subquery clause in a larger query: No implicit Pre-Filter is used. ** `includeTags` and `excludeTags` must not be used in these situations. -The example request below shows two usages of the `knn` query parser that will get _no_ implicit Pre-Filtering from any of the `fq` parameters, because neither usage is as the main `q` param: +The example request below shows two usages of vector query parsers that will get _no_ implicit Pre-Filtering from any of the `fq` parameters, because neither usage is as the main `q` param: [source,text] ---- -?q=(color_str:red OR {!knn f=color_vector topK=10 v="[1.0, 2.0, 3.0, 4.0]"}) +?q=(color_str:red OR {!vectorSimilarity f=color_vector minReturn=0.7 v="[1.0, 2.0, 3.0, 4.0]"}) &fq={!knn f=title_vector topK=10}[9.0, 8.0, 7.0, 6.0] &fq=inStock:true ---- @@ -363,21 +403,24 @@ If we modify the above request to add tags to the `fq` parameters, we can specif &fq={!tag=for_knn}inStock:true ---- -In this example, only the `inStock:true` filter will be used for KNN Pre-Filtering to find the the `topK=10` documents, and the `category:AAA` filter will be applied independently; possibly resulting in less then 10 total matches. +In this example, only the `inStock:true` filter will be used for Pre-Filtering to find the the `topK=10` documents, and the `category:AAA` filter will be applied independently; possibly resulting in less then 10 total matches. Some use cases where `includeTags` and/or `excludeTags` may be more useful then an explicit `preFilter` parameters: -* You have some `fq` parameters that are xref:configuration-guide:requesthandlers-searchcomponents.adoc#paramsets-and-useparams[re-used on many requests] (even when you don't use the `knn` parser) that you wish to be used as KNN Pre-Filters when you _do_ use the `knn` query parser. -* You typically want all `fq` params to be used as KNN Pre-Filters, but when users "drill down" on Facets, you want the `fq` parameters you add to be excluded from the KNN Pre-Filtering so that the result set gets smaller; instead of just computing a new `topK` set. +* You have some `fq` parameters that are xref:configuration-guide:requesthandlers-searchcomponents.adoc#paramsets-and-useparams[re-used on many requests] (even when you don't use search dense vector fields) that you wish to be used as Pre-Filters when you _do_ search dense vector fields. +* You typically want all `fq` params to be used as graph Pre-Filters on your `knn` queries, but when users "drill down" on Facets, you want the `fq` parameters you add to be excluded from the Pre-Filtering so that the result set gets smaller; instead of just computing a new `topK` set. -==== Usage as Re-Ranking Query -The `knn` query parser can be used to rerank first pass query results: +=== Usage in Re-Ranking Query + +Both dense vector search query parsers can be used to rerank first pass query results: + [source,text] &q=id:(3 4 9 2)&rq={!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}&rqq={!knn f=vector topK=10}[1.0, 2.0, 3.0, 4.0] + [IMPORTANT] ==== When using `knn` in re-ranking pay attention to the `topK` parameter. From 59cd484493ea1b4f46b87abc80267c19a823d3e1 Mon Sep 17 00:00:00 2001 From: Nik Osvalds <60047271+nosvalds@users.noreply.github.com> Date: Sat, 22 Jun 2024 10:07:53 -0400 Subject: [PATCH 008/172] add information about large StrField configuration (#1214) --- .../indexing-guide/pages/field-types-included-with-solr.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc index fa4e8855258..81c0601fb6e 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc @@ -75,7 +75,7 @@ Configuration and usage of PreAnalyzedField is documented in the section xref:e |SpatialRecursivePrefixTreeFieldType |(RPT for short) Accepts latitude comma longitude strings or other shapes in WKT format. See xref:query-guide:spatial-search.adoc[] for more information. -|StrField |String (UTF-8 encoded string or Unicode). Strings are intended for small fields and are _not_ tokenized or analyzed in any way. They have a hard limit of slightly less than 32K. +|StrField |String (UTF-8 encoded string or Unicode). Indexed `indexed="true"` strings are intended for small fields and are _not_ tokenized or analyzed in any way. They have a hard limit of slightly less than 32K. Non-indexed `indexed="false"` and non-DocValues `docValues="false"` strings are suitable for storing large strings. |TextField |Text, usually multiple words or tokens. In normal usage, only fields of type TextField or SortableTextField will specify an xref:analyzers.adoc[analyzer]. From 1dfe095fd24f2de497a44b4f7c163c97feb868eb Mon Sep 17 00:00:00 2001 From: Hakim Date: Sat, 22 Jun 2024 16:10:41 +0200 Subject: [PATCH 009/172] Add a note that tells cosine similarities are normalized (#2092) --- .../modules/query-guide/pages/dense-vector-search.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index 35f5f094c68..a165015837a 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -96,6 +96,9 @@ this similarity is intended as an optimized way to perform cosine similarity. In * `cosine`: https://en.wikipedia.org/wiki/Cosine_similarity[Cosine similarity] +[NOTE] +the cosine similarity scores returned by Solr are normalized like this : `(1 + cosine_similarity) / 2`. + [NOTE] the preferred way to perform cosine similarity is to normalize all vectors to unit length, and instead use DOT_PRODUCT. You should only use this function if you need to preserve the original vectors and cannot normalize them in advance. From 7d57f3e481d27a4fda6b5966f48f0492e31561ca Mon Sep 17 00:00:00 2001 From: Carlos Grappa Date: Sat, 22 Jun 2024 12:10:32 -0300 Subject: [PATCH 010/172] SOLR-15591 / Avoid needless catch by checking condition (#340) Co-authored-by: Eric Pugh --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/util/ExternalPaths.java | 26 ++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 44a7af702b1..9ce1c138778 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -133,6 +133,8 @@ Improvements * SOLR-17331: OrderedNodePlacementPlugin will give an even more optimal replica placements during ReplicaMigration commands (Houston Putman, Yohann Callea) +* SOLR-15591: Make using debugger in Solr easier by avoiding NPE in ExternalPaths.determineSourceHome. (@charlygrappa via Eric Pugh) + Optimizations --------------------- * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather diff --git a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java index 122b01829a2..8d21f9e3e84 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java +++ b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java @@ -17,6 +17,7 @@ package org.apache.solr.util; import java.io.File; +import java.net.URL; /** * Some tests need to reach outside the classpath to get certain resources (e.g. the example @@ -28,8 +29,8 @@ public class ExternalPaths { /** * The main directory path for the solr source being built if it can be determined. If it can not - * be determined -- possily because the current context is a client code base using hte test - * frameowrk -- then this variable will be null. + * be determined -- possibly because the current context is a client code base using the test + * framework -- then this variable will be null. * *

Note that all other static paths available in this class are derived from the source home, * and if it is null, those paths will just be relative to 'null' and may not be meaningful. @@ -66,23 +67,24 @@ public class ExternalPaths { */ static String determineSourceHome() { try { - File file; - try { - file = new File("solr/conf"); - if (!file.exists()) { - file = new File(ExternalPaths.class.getClassLoader().getResource("solr/conf").toURI()); + File file = new File("solr/conf"); + if (!file.exists()) { + URL resourceUrl = ExternalPaths.class.getClassLoader().getResource("solr/conf"); + if (resourceUrl != null) { + file = new File(resourceUrl.toURI()); + } else { + // If there is no "solr/conf" in the classpath, fall back to searching from the current + // directory. + file = new File(System.getProperty("tests.src.home", ".")); } - } catch (Exception e) { - // If there is no "solr/conf" in the classpath, fall back to searching from the current - // directory. - file = new File(System.getProperty("tests.src.home", ".")); } + File base = file.getAbsoluteFile(); while (!(new File(base, "solr/CHANGES.txt").exists()) && null != base) { base = base.getParentFile(); } return (null == base) ? null : new File(base, "solr/").getAbsolutePath(); - } catch (RuntimeException e) { + } catch (Exception e) { // all bets are off return null; } From 2193374ef0cffc658e0bd65941cff8b9ba18dc33 Mon Sep 17 00:00:00 2001 From: Pierre Salagnac <82967811+psalagnac@users.noreply.github.com> Date: Sat, 22 Jun 2024 17:12:59 +0200 Subject: [PATCH 011/172] SOLR-17330: When not set, loadOnStartup defaults to true (#2513) --------- Co-authored-by: Eric Pugh --- solr/CHANGES.txt | 2 ++ .../src/java/org/apache/solr/core/CoreDescriptor.java | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9ce1c138778..2ca207bbf42 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -144,6 +144,8 @@ Optimizations * SOLR-17269: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (Patson Luk) +* SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh) + Bug Fixes --------------------- * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java index 5c4744baeba..6109b2f860b 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java +++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java @@ -345,13 +345,13 @@ public CloudDescriptor getCloudDescriptor() { } public boolean isLoadOnStartup() { - String tmp = coreProperties.getProperty(CORE_LOADONSTARTUP, "false"); - return Boolean.parseBoolean(tmp); + String stringValue = coreProperties.getProperty(CORE_LOADONSTARTUP, "true"); + return Boolean.parseBoolean(stringValue); } public boolean isTransient() { - String tmp = coreProperties.getProperty(CORE_TRANSIENT, "false"); - return PropertiesUtil.toBoolean(tmp); + String stringValue = coreProperties.getProperty(CORE_TRANSIENT, "false"); + return PropertiesUtil.toBoolean(stringValue); } public String getUlogDir() { From 79cd576c3a97abe6d80ec6b798dfce7e964f5b1b Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Sat, 22 Jun 2024 16:24:33 +0100 Subject: [PATCH 012/172] SOLR-15582: Remove deprecated APIs in solrj Tuple class. (#257) --- solr/CHANGES.txt | 2 ++ .../java/org/apache/solr/client/solrj/io/Tuple.java | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2ca207bbf42..74f3692b811 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,6 +100,8 @@ Other Changes * SOLR-17279: Introduce SecurityJson.java file to Test Framework to consolidate setting up authentication in tests. (Rudy Seitz via Eric Pugh) +* SOLR-15582: Remove deprecated APIs in solrj Tuple class. (Christine Poerschke via Eric Pugh) + ================== 9.7.0 ================== New Features --------------------- diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/Tuple.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/Tuple.java index fdf992a9ed4..6e2f62ce91c 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/Tuple.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/Tuple.java @@ -49,8 +49,13 @@ public class Tuple implements Cloneable, MapWriter { */ public boolean EXCEPTION; + /** Tuple fields. */ private final Map fields = CollectionUtil.newHashMap(2); + + /** External serializable field names. */ private List fieldNames; + + /** Mapping of external field names to internal tuple field names. */ private Map fieldLabels; public Tuple() { @@ -242,7 +247,7 @@ public void setFieldLabels(Map fieldLabels) { /** * A list of field names to serialize. This list (together with the mapping in {@link - * #getFieldLabels()} determines what tuple values are serialized and their external (serialized) + * #getFieldLabels()}) determines what tuple values are serialized and their external (serialized) * names. * * @return list of external field names or null @@ -279,8 +284,9 @@ public Tuple clone() { } /** - * The other tuples fields and fieldLabels will be putAll'd directly to this's fields and - * fieldLabels while other's fieldNames will be added such that duplicates aren't present. + * The other tuples fields and fieldLabels will be merged via putAll directly into this Tuple's + * fields and fieldLabels while other's fieldNames will be added such that duplicates aren't + * present. * * @param other Tuple to be merged into this. */ From 73b4043a389d3b19daf92cc005d8036b3e33f552 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Mon, 24 Jun 2024 12:26:38 +0100 Subject: [PATCH 013/172] example/films: README.md and vectors tweaks (#2238) --- solr/example/films/README.md | 2 +- solr/example/films/vectors/create_model.py | 2 +- solr/example/films/vectors/films.py | 5 ++++- .../modules/getting-started/pages/tutorial-vectors.adoc | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/solr/example/films/README.md b/solr/example/films/README.md index 0e7229df720..a42e097cc8b 100644 --- a/solr/example/films/README.md +++ b/solr/example/films/README.md @@ -1,6 +1,6 @@ We have a movie data set in JSON, Solr XML, and CSV formats. All 3 formats contain the same data. You can use any one format to index documents to Solr. -This example uses the `_default` configset that ships with Solr plus some custom fields added via Schema API. It demonstrates the use of ParamSets in conjunction with the [Request Parameters API](https://solr.apache.org/guide/solr/latest/configuration-guide/request-parameters-api.html). +This example uses the `_default` configset that ships with Solr plus some custom fields added via Schema API as described [here](https://solr.apache.org/guide/solr/latest/getting-started/tutorial-vectors.html#preparing-for-the-vector-data) in the Solr Reference Guide. It demonstrates the use of ParamSets in conjunction with the [Request Parameters API](https://solr.apache.org/guide/solr/latest/configuration-guide/request-parameters-api.html). The original data was fetched from Freebase and the data license is present in the films-LICENSE.txt file. Freebase was shutdown in 2016 by Google. diff --git a/solr/example/films/vectors/create_model.py b/solr/example/films/vectors/create_model.py index 84a437aa57a..abbd03c2723 100755 --- a/solr/example/films/vectors/create_model.py +++ b/solr/example/films/vectors/create_model.py @@ -50,7 +50,7 @@ model = SentenceTransformer("all-mpnet-base-v2") # New size for the embeddings -new_dimension = 10 +new_dimension = films.FILMS_MODEL_NEW_DIMENSION ######## Evaluate performance of full model ######## diff --git a/solr/example/films/vectors/films.py b/solr/example/films/vectors/films.py index cd1a712eb39..acf211125c7 100644 --- a/solr/example/films/vectors/films.py +++ b/solr/example/films/vectors/films.py @@ -15,13 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import json import csv from lxml import etree from sentence_transformers import SentenceTransformer +FILMS_MODEL_NEW_DIMENSION = int(os.getenv("FILMS_MODEL_NEW_DIMENSION", "10")) + PATH_FILMS_DATASET = "../films.json" -PATH_FILMS_MODEL = "./models/films-model-size_10" +PATH_FILMS_MODEL = f"./models/films-model-size_{FILMS_MODEL_NEW_DIMENSION}" PATH_FILMS_VECTORS_JSON = "./data/films-vectors.json" PATH_FILMS_VECTORS_XML = "./data/films-vectors.xml" PATH_FILMS_VECTORS_CSV = "./data/films-vectors.csv" diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-vectors.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-vectors.adoc index 42a3174658f..63cffc16542 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-vectors.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-vectors.adoc @@ -111,7 +111,7 @@ be used as the input vector for all the following example queries. [NOTE] ==== Interested in calculating the vector using Solr's xref:query-guide:streaming-expressions.adoc[streaming capability]? -Here is an example of a streaming expression that you can run via the Solr Admin Stream UI: +Here is an example of a streaming expression that you can run via the xref:query-guide:stream-screen.adoc[Solr Admin Stream UI]: ``` let( a=select( From 709a1ee27df23b419d09fe8f67c3276409131a4a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 24 Jun 2024 11:03:46 -0400 Subject: [PATCH 014/172] remove changelog, this is just a small refactoring (#2531) --- solr/CHANGES.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 74f3692b811..2ca207bbf42 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,8 +100,6 @@ Other Changes * SOLR-17279: Introduce SecurityJson.java file to Test Framework to consolidate setting up authentication in tests. (Rudy Seitz via Eric Pugh) -* SOLR-15582: Remove deprecated APIs in solrj Tuple class. (Christine Poerschke via Eric Pugh) - ================== 9.7.0 ================== New Features --------------------- From 7b00ddac9ebf129238f700d1812afec57d62b872 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 26 Jun 2024 09:06:44 -0400 Subject: [PATCH 015/172] Fix failing test for SOLR-16962; update log must be enabled (#2537) --- solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java b/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java index 5ca9a2d335b..3c284e8dd12 100644 --- a/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java +++ b/solr/core/src/test/org/apache/solr/update/CustomTLogDirTest.java @@ -78,6 +78,7 @@ public void testIllegalRelative() throws Exception { Path ulogDir = Path.of("../"); Path configSet = LuceneTestCase.createTempDir(); + System.setProperty("enable.update.log", "true"); System.setProperty("solr.test.sys.prop2", "proptwo"); System.setProperty("solr.ulog.dir", ulogDir.toString()); // picked up from `solrconfig.xml` SolrTestCaseJ4.copyMinConf(configSet.toFile(), null, "solrconfig.xml"); From 502faf224ba25a89bd53fb4de9793d5bbb2b2692 Mon Sep 17 00:00:00 2001 From: vinayak hegde Date: Wed, 26 Jun 2024 20:45:32 +0530 Subject: [PATCH 016/172] SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API (#1557) Co-authored-by: Christine Poerschke Co-authored-by: Michael Gibney --- solr/CHANGES.txt | 4 + .../apache/solr/core/QuerySenderListener.java | 11 +- .../org/apache/solr/handler/BlobHandler.java | 4 +- .../solr/handler/MoreLikeThisHandler.java | 7 +- .../handler/admin/IndexSizeEstimator.java | 8 +- .../handler/admin/LukeRequestHandler.java | 11 +- .../component/MoreLikeThisComponent.java | 9 +- .../component/RealTimeGetComponent.java | 4 +- .../component/ResponseLogComponent.java | 7 +- .../component/TermVectorComponent.java | 6 +- .../highlight/DefaultSolrHighlighter.java | 56 ++++---- .../solr/highlight/SolrHighlighter.java | 7 +- .../highlight/UnifiedSolrHighlighter.java | 7 +- .../index/SlowCompositeReaderWrapper.java | 2 + .../apache/solr/response/DocsStreamer.java | 2 +- .../apache/solr/response/ResultContext.java | 11 ++ .../transform/ChildDocTransformer.java | 4 +- .../org/apache/solr/search/ReturnFields.java | 2 +- .../solr/search/SolrDocumentFetcher.java | 124 +++++++++++------- .../apache/solr/search/SolrIndexSearcher.java | 15 ++- .../TopGroupsResultTransformer.java | 12 +- .../org/apache/solr/util/SolrPluginUtils.java | 7 +- .../apache/solr/schema/TestPointFields.java | 4 +- .../org/apache/solr/search/TestDocSet.java | 2 + .../apache/solr/search/TestStressLucene.java | 2 +- .../solr/search/function/TestOrdValues.java | 4 +- .../TestFieldCacheVsDocValues.java | 4 +- .../clustering/ClusteringComponent.java | 4 +- .../solr/ltr/feature/FieldValueFeature.java | 8 +- 29 files changed, 233 insertions(+), 115 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2ca207bbf42..26a88337aac 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -146,6 +146,10 @@ Optimizations * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh) +* SOLR-16677: Update Solr to use new Lucene 9.5 storedFields() API. This removes the use of ThreadLocal for + stored field state, reducing heap usage especially for high-core-count, high-field-count, high-thread-count + cases (Vinayak Hegde, Christine Poerschke, Kevin Risden, David Smiley, Michael Gibney) + Bug Fixes --------------------- * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java index 30c310ca0e3..a612b27809a 100644 --- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java +++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java @@ -30,6 +30,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,13 +81,19 @@ public void close() {} NamedList values = rsp.getValues(); for (int i = 0; i < values.size(); i++) { Object o = values.getVal(i); + SolrDocumentFetcher docFetcher = null; if (o instanceof ResultContext) { - o = ((ResultContext) o).getDocList(); + ResultContext ctx = (ResultContext) o; + o = ctx.getDocList(); + docFetcher = ctx.getDocFetcher(); } if (o instanceof DocList) { DocList docs = (DocList) o; + if (docFetcher == null) { + docFetcher = newSearcher.getDocFetcher(); + } for (DocIterator iter = docs.iterator(); iter.hasNext(); ) { - newSearcher.doc(iter.nextDoc()); + docFetcher.doc(iter.nextDoc()); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java index 3512b0154ed..ff59b99f624 100644 --- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java @@ -140,7 +140,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) long version = 0; if (docs.totalHits.value > 0) { - Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc); + Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc); Number n = doc.getField("version").numericValue(); version = n.longValue(); } @@ -214,7 +214,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) @Override public void write(OutputStream os) throws IOException { - Document doc = req.getSearcher().doc(docs.scoreDocs[0].doc); + Document doc = req.getSearcher().getDocFetcher().doc(docs.scoreDocs[0].doc); IndexableField sf = doc.getField("blob"); FieldType fieldType = req.getSchema().getField("blob").getType(); ByteBuffer buf = (ByteBuffer) fieldType.toObject(sf); diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 2027012d23a..8b4a292b0a1 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -26,9 +26,11 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import net.jcip.annotations.NotThreadSafe; import org.apache.lucene.document.Document; import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; @@ -282,10 +284,12 @@ public static class InterestingTerm { } /** Helper class for MoreLikeThis that can be called from other request handlers */ + @NotThreadSafe public static class MoreLikeThisHelper { final SolrIndexSearcher searcher; final MoreLikeThis mlt; final IndexReader reader; + final StoredFields storedFields; final SchemaField uniqueKeyField; final boolean needDocSet; Map boostFields; @@ -293,6 +297,7 @@ public static class MoreLikeThisHelper { public MoreLikeThisHelper(SolrParams params, SolrIndexSearcher searcher) throws IOException { this.searcher = searcher; this.reader = searcher.getIndexReader(); + this.storedFields = this.reader.storedFields(); this.uniqueKeyField = searcher.getSchema().getUniqueKeyField(); this.needDocSet = params.getBool(FacetParams.FACET, false); @@ -394,7 +399,7 @@ private BooleanQuery getBoostedQuery(Query mltquery) { public DocListAndSet getMoreLikeThis( int id, int start, int rows, List filters, int flags) throws IOException { - Document doc = reader.document(id); + Document doc = this.storedFields.document(id); final Query boostedQuery = getBoostedMLTQuery(id); // exclude current document from results diff --git a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java index b4f222a7a49..87d4764acba 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java @@ -45,6 +45,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -384,11 +386,12 @@ private void estimateTermVectors(Map result) throws IOException for (LeafReaderContext leafReaderContext : reader.leaves()) { LeafReader leafReader = leafReaderContext.reader(); Bits liveDocs = leafReader.getLiveDocs(); + TermVectors leafTermVectors = leafReader.termVectors(); for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) { if (liveDocs != null && !liveDocs.get(docId)) { continue; } - Fields termVectors = leafReader.getTermVectors(docId); + Fields termVectors = leafTermVectors.get(docId); if (termVectors == null) { continue; } @@ -608,11 +611,12 @@ private void estimateStoredFields(Map result) throws IOException mergeInstance.close(); } } else { + StoredFields storedFields = leafReader.storedFields(); for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) { if (liveDocs != null && !liveDocs.get(docId)) { continue; } - leafReader.document(docId, visitor); + storedFields.document(docId, visitor); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index 318f6d0842b..c99c6f9afea 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -52,7 +52,9 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -161,7 +163,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } Document doc = null; try { - doc = reader.document(docId); + doc = reader.storedFields().document(docId); } catch (Exception ex) { } if (doc == null) { @@ -316,6 +318,7 @@ private static SimpleOrderedMap getDocumentFieldsInfo( Document doc, int docId, IndexReader reader, IndexSchema schema) throws IOException { final CharsRefBuilder spare = new CharsRefBuilder(); SimpleOrderedMap finfo = new SimpleOrderedMap<>(); + TermVectors termVectors = null; for (Object o : doc.getFields()) { Field field = (Field) o; SimpleOrderedMap f = new SimpleOrderedMap<>(); @@ -353,7 +356,8 @@ private static SimpleOrderedMap getDocumentFieldsInfo( // If we have a term vector, return that if (field.fieldType().storeTermVectors()) { try { - Terms v = reader.getTermVector(docId, field.name()); + if (termVectors == null) termVectors = reader.termVectors(); + Terms v = termVectors.get(docId, field.name()); if (v != null) { SimpleOrderedMap tfv = new SimpleOrderedMap<>(); final TermsEnum termsEnum = v.iterator(); @@ -462,6 +466,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I PostingsEnum postingsEnum = null; TermsEnum termsEnum = terms.iterator(); BytesRef text; + StoredFields storedFields = reader.storedFields(); // Deal with the chance that the first bunch of terms are in deleted documents. Is there a // better way? for (int idx = 0; idx < 1000 && postingsEnum == null; ++idx) { @@ -476,7 +481,7 @@ private static Document getFirstLiveDoc(Terms terms, LeafReader reader) throws I if (liveDocs != null && liveDocs.get(postingsEnum.docID())) { continue; } - return reader.document(postingsEnum.docID()); + return storedFields.document(postingsEnum.docID()); } } return null; diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index bc1267bfd54..2b41bdd1245 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -49,6 +49,7 @@ import org.apache.solr.search.DocListAndSet; import org.apache.solr.search.QueryLimits; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.slf4j.Logger; @@ -103,6 +104,7 @@ public void process(ResponseBuilder rb) throws IOException { new MoreLikeThisHandler.MoreLikeThisHelper(params, searcher); NamedList> mltQueryByDocKey = new NamedList<>(); QueryLimits queryLimits = QueryLimits.getCurrentLimits(); + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); for (DocIterator results = rb.getResults().docList.iterator(); results.hasNext(); ) { int docId = results.nextDoc(); final List interestingTerms = @@ -114,7 +116,7 @@ public void process(ResponseBuilder rb) throws IOException { break; } final String uniqueKey = rb.req.getSchema().getUniqueKeyField().getName(); - final Document document = rb.req.getSearcher().doc(docId); + final Document document = docFetcher.doc(docId); final String uniqueVal = rb.req.getSchema().printableUniqueKey(document); final NamedList mltQ = mltViaQueryParams(rb.req.getSchema(), interestingTerms, uniqueKey, uniqueVal); @@ -423,12 +425,13 @@ NamedList getMoreLikeThese( QueryLimits queryLimits = QueryLimits.getCurrentLimits(); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); while (iterator.hasNext()) { int id = iterator.nextDoc(); int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5); DocListAndSet similarDocuments = mltHelper.getMoreLikeThis(id, 0, rows, null, flags); - String name = schema.printableUniqueKey(searcher.doc(id)); + String name = schema.printableUniqueKey(docFetcher.doc(id)); mltResponse.add(name, similarDocuments.docList); if (dbg != null) { @@ -440,7 +443,7 @@ NamedList getMoreLikeThese( DocIterator similarDocumentsIterator = similarDocuments.docList.iterator(); while (similarDocumentsIterator.hasNext()) { int mltid = similarDocumentsIterator.nextDoc(); - String key = schema.printableUniqueKey(searcher.doc(mltid)); + String key = schema.printableUniqueKey(docFetcher.doc(mltid)); explains.add(key, searcher.explain(mltHelper.getRealMLTQuery(), mltid)); } docDbg.add("explain", explains); diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index 33fa334758a..f1fc77d9ceb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -347,10 +347,10 @@ public void process(ResponseBuilder rb) throws IOException { if (docid < 0) continue; + SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher(); Document luceneDocument = - searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames()); + docFetcher.doc(docid, rsp.getReturnFields().getLuceneFieldNames()); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); - SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher(); if (reuseDvIters == null) { reuseDvIters = new DocValuesIteratorCache(searcherInfo.getSearcher()); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java index eba622615a3..6d8ee1957fb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseLogComponent.java @@ -24,6 +24,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; /** @@ -90,9 +91,10 @@ protected void processIds( StringBuilder sb = new StringBuilder(); Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { - sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); + sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))).append(','); } if (sb.length() > 0) { rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1)); @@ -105,8 +107,9 @@ protected void processScores( StringBuilder sb = new StringBuilder(); Set fields = Collections.singleton(schema.getUniqueKeyField().getName()); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { - sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))) + sb.append(schema.printableUniqueKey(docFetcher.doc(iter.nextDoc(), fields))) .append(':') .append(iter.score()) .append(','); diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java index 6373b0a6337..73eaa3b817b 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -251,6 +252,7 @@ public void process(ResponseBuilder rb) throws IOException { SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader reader = searcher.getIndexReader(); + TermVectors termVectors = reader.termVectors(); // the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors // Only load the id field to get the uniqueKey of that @@ -277,7 +279,7 @@ public void process(ResponseBuilder rb) throws IOException { if (null != fields) { for (Map.Entry entry : fieldOptions.entrySet()) { final String field = entry.getKey(); - final Terms vector = reader.getTermVector(docId, field); + final Terms vector = termVectors.get(docId, field); if (vector != null) { TermsEnum termsEnum = vector.iterator(); mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field); @@ -285,7 +287,7 @@ public void process(ResponseBuilder rb) throws IOException { } } else { // extract all fields - final Fields vectors = reader.getTermVectors(docId); + final Fields vectors = termVectors.get(docId); // There can be no documents with vectors if (vectors != null) { for (String field : vectors) { diff --git a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java index 6fc2d475942..ba22b2e317c 100644 --- a/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java @@ -33,10 +33,9 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.index.Fields; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -78,6 +77,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.plugin.PluginInfoInitialized; @@ -482,15 +482,16 @@ public NamedList doHighlighting( // Lazy container for fvh and fieldQuery FvhContainer fvhContainer = new FvhContainer(null, null); - IndexReader reader = - new TermVectorReusingLeafReader(req.getSearcher().getSlowAtomicReader()); // SOLR-5855 + IndexReader reader = req.getSearcher().getSlowAtomicReader(); + TermVectors termVectors = new ReusingTermVectors(reader); // SOLR-5855 // Highlight each document NamedList fragments = new SimpleOrderedMap<>(); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); DocIterator iterator = docs.iterator(); for (int i = 0; i < docs.size(); i++) { int docId = iterator.nextDoc(); - SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, returnFields); + SolrDocument doc = docFetcher.solrDoc(docId, returnFields); NamedList docHighlights = new SimpleOrderedMap<>(); // Highlight per-field @@ -500,10 +501,11 @@ public NamedList doHighlighting( Object fieldHighlights; // object type allows flexibility for subclassers fieldHighlights = doHighlightingOfField( - doc, docId, schemaField, fvhContainer, query, reader, req, params); + doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params); if (fieldHighlights == null) { - fieldHighlights = alternateField(doc, docId, fieldName, fvhContainer, query, reader, req); + fieldHighlights = + alternateField(doc, docId, fieldName, fvhContainer, query, reader, termVectors, req); } if (fieldHighlights != null) { @@ -522,6 +524,7 @@ protected Object doHighlightingOfField( FvhContainer fvhContainer, Query query, IndexReader reader, + TermVectors termVectors, SolrQueryRequest req, SolrParams params) throws IOException { @@ -570,7 +573,8 @@ protected void flatten( fieldHighlights = doHighlightingByFastVectorHighlighter(doc, docId, schemaField, fvhContainer, reader, req); } else { // standard/default highlighter - fieldHighlights = doHighlightingByHighlighter(doc, docId, schemaField, query, reader, req); + fieldHighlights = + doHighlightingByHighlighter(doc, docId, schemaField, query, termVectors, req); } return fieldHighlights; } @@ -656,7 +660,7 @@ protected Object doHighlightingByHighlighter( int docId, SchemaField schemaField, Query query, - IndexReader reader, + TermVectors termVectors, SolrQueryRequest req) throws IOException { final SolrParams params = req.getParams(); @@ -696,7 +700,7 @@ protected Object doHighlightingByHighlighter( // Try term vectors, which is faster // note: offsets are minimally sufficient for this HL. - final Fields tvFields = schemaField.storeTermOffsets() ? reader.getTermVectors(docId) : null; + final Fields tvFields = schemaField.storeTermOffsets() ? termVectors.get(docId) : null; final TokenStream tvStream = TokenSources.getTermVectorTokenStreamOrNull(fieldName, tvFields, maxCharsToAnalyze - 1); // We need to wrap in OffsetWindowTokenFilter if multi-valued @@ -858,6 +862,7 @@ protected Object alternateField( FvhContainer fvhContainer, Query query, IndexReader reader, + TermVectors termVectors, SolrQueryRequest req) throws IOException { IndexSchema schema = req.getSearcher().getSchema(); @@ -887,7 +892,7 @@ protected Object alternateField( req.setParams(SolrParams.wrapDefaults(new MapSolrParams(invariants), origParams)); fieldHighlights = doHighlightingOfField( - doc, docId, schemaField, fvhContainer, query, reader, req, params); + doc, docId, schemaField, fvhContainer, query, reader, termVectors, req, params); req.setParams(origParams); if (fieldHighlights != null) { return fieldHighlights; @@ -1087,35 +1092,30 @@ public boolean incrementToken() throws IOException { } /** - * Wraps a DirectoryReader that caches the {@link LeafReader#getTermVectors(int)} so that if the - * next call has the same ID, then it is reused. + * Wraps a TermVectors and caches the {@link TermVectors#get(int)} so that if the next call has + * the same ID, then it is reused. */ - static class TermVectorReusingLeafReader extends FilterLeafReader { + static class ReusingTermVectors extends TermVectors { + private final IndexReader reader; + private TermVectors in; private int lastDocId = -1; private Fields tvFields; - public TermVectorReusingLeafReader(LeafReader in) { - super(in); + public ReusingTermVectors(IndexReader reader) { + this.reader = reader; } @Override - public Fields getTermVectors(int docID) throws IOException { + public Fields get(int docID) throws IOException { if (docID != lastDocId) { + if (in == null) { + in = reader.termVectors(); + } lastDocId = docID; - tvFields = in.getTermVectors(docID); + tvFields = in.get(docID); } return tvFields; } - - @Override - public CacheHelper getCoreCacheHelper() { - return null; - } - - @Override - public CacheHelper getReaderCacheHelper() { - return null; - } } } diff --git a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java index 4e7729fa49a..e106c471f63 100644 --- a/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/SolrHighlighter.java @@ -29,6 +29,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.util.SolrPluginUtils; public abstract class SolrHighlighter { @@ -73,7 +74,11 @@ public String[] getHighlightFields( } else { fields = expandWildcardsInFields( - () -> request.getSearcher().getDocFetcher().getStoredHighlightFieldNames(), fields); + () -> + request + .getSearcher() + .interrogateDocFetcher(SolrDocumentFetcher::getStoredHighlightFieldNames), + fields); } // Trim them now in case they haven't been yet. Not needed for all code-paths above but do it diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java index f14c43d1fd3..4390893ec76 100644 --- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java @@ -49,6 +49,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.util.RTimerTree; @@ -223,10 +224,11 @@ protected String[] getUniqueKeys(SolrIndexSearcher searcher, int[] docIDs) throw SchemaField keyField = schema.getUniqueKeyField(); if (keyField != null) { SolrReturnFields returnFields = new SolrReturnFields(keyField.getName(), null); + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); String[] uniqueKeys = new String[docIDs.length]; for (int i = 0; i < docIDs.length; i++) { int docid = docIDs[i]; - SolrDocument solrDoc = searcher.getDocFetcher().solrDoc(docid, returnFields); + SolrDocument solrDoc = docFetcher.solrDoc(docid, returnFields); uniqueKeys[i] = schema.printableUniqueKey(solrDoc); } return uniqueKeys; @@ -446,7 +448,8 @@ protected Predicate getFieldMatcher(String field) { if (queryFieldPattern != null && queryFieldPattern.length != 0) { Supplier> indexedFieldsSupplier = - () -> solrIndexSearcher.getDocFetcher().getIndexedFieldNames(); + () -> + solrIndexSearcher.interrogateDocFetcher(SolrDocumentFetcher::getIndexedFieldNames); Set fields = Set.of(expandWildcardsInFields(indexedFieldsSupplier, queryFieldPattern)); diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java index f8fffe9bce8..cdfbb181671 100644 --- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java +++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java @@ -313,6 +313,7 @@ public NumericDocValues getNormValues(String field) throws IOException { } @Override + @Deprecated public Fields getTermVectors(int docID) throws IOException { return in.getTermVectors(docID); } @@ -342,6 +343,7 @@ public int maxDoc() { } @Override + @Deprecated public void document(int docID, StoredFieldVisitor visitor) throws IOException { ensureOpen(); in.document(docID, visitor); diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index 8fdcefa3d3f..485d9c9a1e8 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -73,7 +73,7 @@ public DocsStreamer(ResultContext rctx) { this.docs = rctx.getDocList(); transformer = rctx.getReturnFields().getTransformer(); docIterator = this.docs.iterator(); - docFetcher = rctx.getSearcher().getDocFetcher(); + docFetcher = rctx.getDocFetcher(); solrReturnFields = (SolrReturnFields) rctx.getReturnFields(); if (transformer != null) transformer.setContext(rctx); diff --git a/solr/core/src/java/org/apache/solr/response/ResultContext.java b/solr/core/src/java/org/apache/solr/response/ResultContext.java index a74d0484d08..ea07d076da5 100644 --- a/solr/core/src/java/org/apache/solr/response/ResultContext.java +++ b/solr/core/src/java/org/apache/solr/response/ResultContext.java @@ -23,17 +23,28 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; /** A class to hold the QueryResult and the Query */ public abstract class ResultContext { + private SolrDocumentFetcher docFetcher; + public abstract DocList getDocList(); public abstract ReturnFields getReturnFields(); public abstract SolrIndexSearcher getSearcher(); + public SolrDocumentFetcher getDocFetcher() { + if (docFetcher == null) { + return docFetcher = getSearcher().getDocFetcher(); + } else { + return docFetcher; + } + } + public abstract Query getQuery(); // TODO: any reason to allow for retrieval of any filters as well? diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java index 7eaac00e52c..d8a4f084226 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java @@ -45,6 +45,7 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.BitsFilteredPostingsEnum; import org.apache.solr.search.DocSet; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.slf4j.Logger; @@ -173,6 +174,7 @@ public void transform(SolrDocument rootDoc, int rootDocId) { final Map>> pendingParentPathsToChildren = new HashMap<>(); + SolrDocumentFetcher docFetcher = context.getDocFetcher(); final int firstChildId = segBaseId + segPrevRootId + 1; int matches = 0; // Loop each child ID up to the parent (exclusive). @@ -207,7 +209,7 @@ public void transform(SolrDocument rootDoc, int rootDocId) { ++matches; // note: includes ancestors that are not necessarily in childDocSet // load the doc - SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, childReturnFields); + SolrDocument doc = docFetcher.solrDoc(docId, childReturnFields); if (childReturnFields.getTransformer() != null) { if (childReturnFields.getTransformer().context == null) { childReturnFields.getTransformer().setContext(context); diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java index e93b0fae772..44dcb12491e 100644 --- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java @@ -28,7 +28,7 @@ public abstract class ReturnFields { /** * Set of field names with their exact names from the lucene index. Class such as ResponseWriters - * pass this to {@link SolrIndexSearcher#doc(int, Set)}. + * pass this to {@link SolrDocumentFetcher#doc(int, Set)}. * *

NOTE: In some situations, this method may return null even if {@link * #wantsAllFields()} is false. For example: When glob expressions are used ({@link diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java index e6ccb9edd18..ea7cf854aab 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.UncheckedIOException; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -42,7 +43,6 @@ import org.apache.lucene.document.StoredValue; import org.apache.lucene.document.TextField; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; @@ -57,6 +57,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.misc.document.LazyDocument; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -114,11 +115,38 @@ public class SolrDocumentFetcher { private final Set largeFields; - private Collection storedHighlightFieldNames; // lazy populated; use getter + private final Collection[] storedHighlightFieldNames; // lazy populated; use getter + + private final Collection[] indexedFieldNames; // lazy populated; use getter + + private final StoredFields storedFields; + + private SolrDocumentFetcher(SolrDocumentFetcher template, StoredFields storedFields) { + this.searcher = template.searcher; + this.nLeaves = template.nLeaves; + this.enableLazyFieldLoading = template.enableLazyFieldLoading; + this.documentCache = template.documentCache; + this.nonStoredDVsUsedAsStored = template.nonStoredDVsUsedAsStored; + this.allNonStoredDVs = template.allNonStoredDVs; + this.nonStoredDVsWithoutCopyTargets = template.nonStoredDVsWithoutCopyTargets; + this.largeFields = template.largeFields; + this.dvsCanSubstituteStored = template.dvsCanSubstituteStored; + this.allStored = template.allStored; + this.storedHighlightFieldNames = template.indexedFieldNames; + this.indexedFieldNames = template.indexedFieldNames; + this.storedFields = storedFields; + } - private Collection indexedFieldNames; // lazy populated; use getter + @Override + protected SolrDocumentFetcher clone() { + try { + return new SolrDocumentFetcher(this, searcher.getIndexReader().storedFields()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - @SuppressWarnings({"unchecked"}) + @SuppressWarnings({"unchecked", "rawtypes"}) SolrDocumentFetcher(SolrIndexSearcher searcher, SolrConfig solrConfig, boolean cachingEnabled) { this.searcher = searcher; this.nLeaves = searcher.getTopReaderContext().leaves().size(); @@ -172,6 +200,9 @@ public class SolrDocumentFetcher { this.largeFields = Collections.unmodifiableSet(storedLargeFields); this.dvsCanSubstituteStored = Collections.unmodifiableSet(dvsCanSubstituteStored); this.allStored = Collections.unmodifiableSet(allStoreds); + this.storedFields = null; // template docFetcher should throw NPE if used directly + this.storedHighlightFieldNames = new Collection[1]; + this.indexedFieldNames = new Collection[1]; } // Does this field have both stored=true and docValues=true and is otherwise @@ -203,9 +234,9 @@ public SolrCache getDocumentCache() { * reader knows about. */ public Collection getStoredHighlightFieldNames() { - synchronized (this) { - if (storedHighlightFieldNames == null) { - storedHighlightFieldNames = new ArrayList<>(); + synchronized (storedHighlightFieldNames) { + if (storedHighlightFieldNames[0] == null) { + Collection storedHighlightFieldNames = new ArrayList<>(); for (FieldInfo fieldInfo : searcher.getFieldInfos()) { final String fieldName = fieldInfo.name; try { @@ -220,23 +251,25 @@ public Collection getStoredHighlightFieldNames() { log.warn("Field [{}] found in index, but not defined in schema.", fieldName); } } + this.storedHighlightFieldNames[0] = storedHighlightFieldNames; } - return storedHighlightFieldNames; + return storedHighlightFieldNames[0]; } } /** Returns a collection of the names of all indexed fields which the index reader knows about. */ public Collection getIndexedFieldNames() { - synchronized (this) { - if (indexedFieldNames == null) { - indexedFieldNames = new ArrayList<>(); + synchronized (indexedFieldNames) { + if (indexedFieldNames[0] == null) { + Collection indexedFieldNames = new ArrayList<>(); for (FieldInfo fieldInfo : searcher.getFieldInfos()) { if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { indexedFieldNames.add(fieldInfo.name); } } + this.indexedFieldNames[0] = indexedFieldNames; } - return indexedFieldNames; + return indexedFieldNames[0]; } } @@ -272,10 +305,9 @@ public Document doc(int i, Set fields) throws IOException { } private Document docNC(int i, Set fields) throws IOException { - final DirectoryReader reader = searcher.getIndexReader(); final SolrDocumentStoredFieldVisitor visitor = - new SolrDocumentStoredFieldVisitor(fields, reader, i); - reader.document(i, visitor); + new SolrDocumentStoredFieldVisitor(fields, searcher.getIndexReader(), i); + storedFields.document(i, visitor); return visitor.getDocument(); } @@ -368,16 +400,14 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { } } - /** - * @see SolrIndexSearcher#doc(int, StoredFieldVisitor) - */ + /** Visit a document's fields using a {@link StoredFieldVisitor}. */ public void doc(int docId, StoredFieldVisitor visitor) throws IOException { if (documentCache != null) { // get cached document or retrieve it including all fields (and cache it) Document cached = doc(docId); visitFromCached(cached, visitor); } else { - searcher.getIndexReader().document(docId, visitor); + storedFields.document(docId, visitor); } } @@ -493,35 +523,33 @@ synchronized BytesRef readBytes() throws IOException { return cachedBytes; } else { BytesRef bytesRef = new BytesRef(); - searcher - .getIndexReader() - .document( - docId, - new StoredFieldVisitor() { - boolean done = false; - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (done) { - return Status.STOP; - } - return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) throws IOException { - Objects.requireNonNull(value, "String value should not be null"); - bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8); - bytesRef.length = bytesRef.bytes.length; - done = true; - } - - @Override - public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { - throw new UnsupportedOperationException( - "'large' binary fields are not (yet) supported"); - } - }); + storedFields.document( + docId, + new StoredFieldVisitor() { + boolean done = false; + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (done) { + return Status.STOP; + } + return fieldInfo.name.equals(name()) ? Status.YES : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + Objects.requireNonNull(value, "String value should not be null"); + bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8); + bytesRef.length = bytesRef.bytes.length; + done = true; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + throw new UnsupportedOperationException( + "'large' binary fields are not (yet) supported"); + } + }); if (bytesRef.length < largeValueLengthCacheThreshold) { return cachedBytes = bytesRef; } else { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index d4d1f64bcda..a8bc38b5699 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -427,7 +428,16 @@ public SolrIndexSearcher( } public SolrDocumentFetcher getDocFetcher() { - return docFetcher; + return docFetcher.clone(); + } + + /** + * Allows interrogation of {@link #docFetcher} template (checking field names, etc.) without + * forcing it to be cloned (as it would be if an instance were retrieved via {@link + * #getDocFetcher()}). + */ + public T interrogateDocFetcher(Function func) { + return func.apply(docFetcher); } public StatsCache getStatsCache() { @@ -755,6 +765,7 @@ public LimitExceededFromScorerException(String msg) { * @see SolrDocumentFetcher */ @Override + @Deprecated public Document doc(int docId) throws IOException { return doc(docId, (Set) null); } @@ -767,6 +778,7 @@ public Document doc(int docId) throws IOException { * @see SolrDocumentFetcher */ @Override + @Deprecated public final void doc(int docId, StoredFieldVisitor visitor) throws IOException { getDocFetcher().doc(docId, visitor); } @@ -780,6 +792,7 @@ public final void doc(int docId, StoredFieldVisitor visitor) throws IOException * @see SolrDocumentFetcher */ @Override + @Deprecated public final Document doc(int i, Set fields) throws IOException { return getDocFetcher().doc(i, fields); } diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java index e385fcb941d..83255675fdd 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java @@ -43,6 +43,7 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.grouping.Command; import org.apache.solr.search.grouping.distributed.command.QueryCommand; import org.apache.solr.search.grouping.distributed.command.QueryCommandResult; @@ -229,12 +230,13 @@ protected NamedList serializeTopGroups(TopGroups data, SchemaF groupResult.add("maxScore", searchGroup.maxScore); } + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); List> documents = new ArrayList<>(); for (int i = 0; i < searchGroup.scoreDocs.length; i++) { NamedList document = new NamedList<>(); documents.add(document); - Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc); + Document doc = retrieveDocument(uniqueField, searchGroup.scoreDocs[i].doc, docFetcher); document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName()))); if (!Float.isNaN(searchGroup.scoreDocs[i].score)) { document.add("score", searchGroup.scoreDocs[i].score); @@ -290,13 +292,14 @@ protected NamedList serializeTopDocs(QueryCommandResult result) throws I List> documents = new ArrayList<>(); queryResult.add("documents", documents); + SolrDocumentFetcher docFetcher = rb.req.getSearcher().getDocFetcher(); final IndexSchema schema = rb.req.getSearcher().getSchema(); SchemaField uniqueField = schema.getUniqueKeyField(); for (ScoreDoc scoreDoc : result.getTopDocs().scoreDocs) { NamedList document = new NamedList<>(); documents.add(document); - Document doc = retrieveDocument(uniqueField, scoreDoc.doc); + Document doc = retrieveDocument(uniqueField, scoreDoc.doc, docFetcher); document.add(ID, uniqueField.getType().toExternal(doc.getField(uniqueField.getName()))); if (!Float.isNaN(scoreDoc.score)) { document.add("score", scoreDoc.score); @@ -322,7 +325,8 @@ protected NamedList serializeTopDocs(QueryCommandResult result) throws I return queryResult; } - private Document retrieveDocument(final SchemaField uniqueField, int doc) throws IOException { - return rb.req.getSearcher().doc(doc, Collections.singleton(uniqueField.getName())); + private Document retrieveDocument( + final SchemaField uniqueField, int doc, SolrDocumentFetcher docFetcher) throws IOException { + return docFetcher.doc(doc, Collections.singleton(uniqueField.getName())); } } diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index 3b564972eb7..a76ebe237a3 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -76,6 +76,7 @@ import org.apache.solr.search.QParser; import org.apache.solr.search.QueryParsing; import org.apache.solr.search.ReturnFields; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrQueryParser; import org.apache.solr.search.SortSpecParsing; @@ -249,9 +250,10 @@ public static void optimizePreFetchDocs( } // get documents + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); DocIterator iter = docs.iterator(); for (int i = 0; i < docs.size(); i++) { - searcher.doc(iter.nextDoc(), fieldFilter); + docFetcher.doc(iter.nextDoc(), fieldFilter); } } } @@ -404,12 +406,13 @@ public static NamedList getExplanations( Query query, DocList docs, SolrIndexSearcher searcher, IndexSchema schema) throws IOException { + SolrDocumentFetcher docFetcher = searcher.getDocFetcher(); NamedList explainList = new SimpleOrderedMap<>(); DocIterator iterator = docs.iterator(); for (int i = 0; i < docs.size(); i++) { int id = iterator.nextDoc(); - Document doc = searcher.doc(id); + Document doc = docFetcher.doc(id); String strid = schema.printableUniqueKey(doc); explainList.add(strid, searcher.explain(query, id)); diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java index ffb6f69ee29..f92fbe71e03 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java +++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java @@ -56,6 +56,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.PointRangeQuery; @@ -5811,8 +5812,9 @@ private void doTestInternals(String field, String[] values) throws IOException { } for (LeafReaderContext leave : ir.leaves()) { LeafReader reader = leave.reader(); + StoredFields storedFields = reader.storedFields(); for (int i = 0; i < reader.numDocs(); i++) { - Document doc = reader.document(i); + Document doc = storedFields.document(i); if (sf.stored()) { assertNotNull("Field " + field + " not found. Doc: " + doc, doc.get(field)); } else { diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java index 901a188fedd..a554ce2f409 100644 --- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java +++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java @@ -303,6 +303,7 @@ public int maxDoc() { } @Override + @Deprecated public Fields getTermVectors(int docID) { return null; } @@ -394,6 +395,7 @@ public void searchNearestVectors( protected void doClose() {} @Override + @Deprecated public void document(int doc, StoredFieldVisitor visitor) {} @Override diff --git a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java index a02b55cb92f..2fc620f8468 100644 --- a/solr/core/src/test/org/apache/solr/search/TestStressLucene.java +++ b/solr/core/src/test/org/apache/solr/search/TestStressLucene.java @@ -347,7 +347,7 @@ public void run() { verbose("ERROR: Couldn't find a doc for id", id, "using reader", r); } assertTrue(docid >= 0); // we should have found the document, or its tombstone - Document doc = r.document(docid); + Document doc = r.storedFields().document(docid); long foundVal = Long.parseLong(doc.get(FIELD)); if (foundVal < Math.abs(val)) { verbose( diff --git a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java index 5c37805b9e4..4081fc3b671 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.IndexSearcher; @@ -137,9 +138,10 @@ private void doTestExactScore(String field, boolean inOrder) throws Exception { TopDocs td = s.search(q, 1000); assertEquals("All docs should be matched!", N_DOCS, td.totalHits.value); ScoreDoc sd[] = td.scoreDocs; + StoredFields storedFields = s.getIndexReader().storedFields(); for (int i = 0; i < sd.length; i++) { float score = sd[i].score; - String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD); + String id = storedFields.document(sd[i].doc).get(ID_FIELD); log("-------- " + i + ". Explain doc " + id); log(s.explain(q, sd[i].doc)); float expectedScore = N_DOCS - i - 1; diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java index 89ae4181f85..ec1525cef7d 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; @@ -174,8 +175,9 @@ public void testHugeBinaryValues() throws Exception { TestUtil.checkReader(ar); BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field"); + StoredFields storedFields = ar.storedFields(); for (int docID = 0; docID < docBytes.size(); docID++) { - Document doc = ar.document(docID); + Document doc = storedFields.document(docID); assertEquals(docID, s.nextDoc()); BytesRef bytes = s.binaryValue(); byte[] expected = docBytes.get(Integer.parseInt(doc.get("id"))); diff --git a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java index d18e1b7c924..27646190a39 100644 --- a/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java +++ b/solr/modules/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java @@ -55,6 +55,7 @@ import org.apache.solr.search.DocList; import org.apache.solr.search.DocSlice; import org.apache.solr.search.QueryLimits; +import org.apache.solr.search.SolrDocumentFetcher; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.plugin.SolrCoreAware; import org.carrot2.clustering.Cluster; @@ -415,13 +416,14 @@ public SolrIndexSearcher getSearcher() { docLanguage = (doc) -> requestParameters.language(); } + SolrDocumentFetcher docFetcher = indexSearcher.getDocFetcher(); List result = new ArrayList<>(); DocIterator it = responseBuilder.getResults().docList.iterator(); while (it.hasNext()) { int docId = it.nextDoc(); Map docFieldValues = new LinkedHashMap<>(); - for (IndexableField indexableField : indexSearcher.doc(docId, fieldsToLoad.keySet())) { + for (IndexableField indexableField : docFetcher.doc(docId, fieldsToLoad.keySet())) { String fieldName = indexableField.name(); Function toString = fieldsToLoad.get(fieldName); if (toString != null) { diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java index 583e080954c..02e2fabb5bf 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/FieldValueFeature.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -183,18 +184,21 @@ public FeatureScorer scorer(LeafReaderContext context) throws IOException { public class FieldValueFeatureScorer extends FeatureScorer { private final LeafReaderContext context; + private final StoredFields storedFields; public FieldValueFeatureScorer( - FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr) { + FeatureWeight weight, LeafReaderContext context, DocIdSetIterator itr) + throws IOException { super(weight, itr); this.context = context; + this.storedFields = (context == null ? null : context.reader().storedFields()); } @Override public float score() throws IOException { try { - final Document document = context.reader().document(itr.docID(), fieldAsSet); + final Document document = storedFields.document(itr.docID(), fieldAsSet); final IndexableField indexableField = document.getField(field); if (indexableField == null) { return getDefaultValue(); From 390c30ff56ad354a6ee55eaae54713dd8ec4cce3 Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Wed, 26 Jun 2024 19:54:20 -0400 Subject: [PATCH 017/172] SOLR-17349: SolrDocumentFetcher should always skip lazy field loading overhead if documentCache==null (#2535) --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/search/SolrDocumentFetcher.java | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 26a88337aac..6c89c9867ee 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -150,6 +150,8 @@ Optimizations stored field state, reducing heap usage especially for high-core-count, high-field-count, high-thread-count cases (Vinayak Hegde, Christine Poerschke, Kevin Risden, David Smiley, Michael Gibney) +* SOLR-17349: SolrDocumentFetcher should always skip lazy field loading overhead if documentCache==null (Michael Gibney) + Bug Fixes --------------------- * SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh) diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java index ea7cf854aab..4526d437a6e 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -359,9 +359,15 @@ private class SolrDocumentStoredFieldVisitor extends DocumentStoredFieldVisitor super(toLoad); this.docId = docId; this.doc = getDocument(); - this.lazyFieldProducer = - toLoad != null && enableLazyFieldLoading ? new LazyDocument(reader, docId) : null; - this.addLargeFieldsLazily = (documentCache != null && !largeFields.isEmpty()); + if (documentCache == null) { + // lazy loading makes no sense if we don't have a `documentCache` + this.lazyFieldProducer = null; + this.addLargeFieldsLazily = false; + } else { + this.lazyFieldProducer = + toLoad != null && enableLazyFieldLoading ? new LazyDocument(reader, docId) : null; + this.addLargeFieldsLazily = !largeFields.isEmpty(); + } // TODO can we return Status.STOP after a val is loaded and we know there are no other fields // of interest? // When: toLoad is one single-valued field, no lazyFieldProducer From 0df3e63ffbdcde0eb953876a3fb47994d334d8f2 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 27 Jun 2024 12:34:50 -0400 Subject: [PATCH 018/172] SOLR-16824: Adopt Linux Command line tool pattern of -- for long option commands (#1768) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This goes through all the various CLI tools and moves them all to using the same long form across all our tools, replacing a mish mash of older formats. -zkHost is now --zk-host for example. This pattern applies to both Windows and Linux bin/solr commands. The old forms are now deprecated, though they continue to work. The -u short command for --credentials is now used everywhere, this did mean the Assert tools "--same-user" command no longer has a "-u" option. --------- Co-authored-by: Jan Høydahl Co-authored-by: Solr Bot <125606113+solrbot@users.noreply.github.com> Co-authored-by: Jan Høydahl Co-authored-by: Jason Gerlowski --- dev-docs/FAQ.adoc | 3 +- dev-tools/scripts/cloud.sh | 2 +- solr/CHANGES.txt | 4 + solr/bin/solr | 213 +++++++------ solr/bin/solr.cmd | 119 +++---- solr/core/build.gradle | 2 - .../src/java/org/apache/solr/cli/ApiTool.java | 20 +- .../java/org/apache/solr/cli/AssertTool.java | 15 +- .../java/org/apache/solr/cli/AuthTool.java | 71 ++--- .../java/org/apache/solr/cli/ClusterTool.java | 7 +- .../solr/cli/ConfigSetDownloadTool.java | 22 +- .../apache/solr/cli/ConfigSetUploadTool.java | 32 +- .../java/org/apache/solr/cli/ConfigTool.java | 11 +- .../java/org/apache/solr/cli/CreateTool.java | 100 ++++-- .../java/org/apache/solr/cli/DeleteTool.java | 53 +++- .../java/org/apache/solr/cli/ExportTool.java | 16 +- .../org/apache/solr/cli/HealthcheckTool.java | 11 +- .../org/apache/solr/cli/LinkConfigTool.java | 7 +- .../java/org/apache/solr/cli/PackageTool.java | 57 ++-- .../org/apache/solr/cli/PostLogsTool.java | 6 +- .../java/org/apache/solr/cli/PostTool.java | 106 ++++--- .../org/apache/solr/cli/RunExampleTool.java | 103 +++--- .../src/java/org/apache/solr/cli/SolrCLI.java | 88 ++++-- .../java/org/apache/solr/cli/StatusTool.java | 13 +- .../org/apache/solr/cli/UpdateACLTool.java | 3 +- .../java/org/apache/solr/cli/ZkCpTool.java | 20 +- .../java/org/apache/solr/cli/ZkLsTool.java | 10 +- .../org/apache/solr/cli/ZkMkrootTool.java | 8 +- .../java/org/apache/solr/cli/ZkMvTool.java | 20 +- .../java/org/apache/solr/cli/ZkRmTool.java | 10 +- .../org/apache/solr/cli/AuthToolTest.java | 8 +- .../org/apache/solr/cli/CreateToolTest.java | 2 +- .../org/apache/solr/cli/DeleteToolTest.java | 6 +- .../apache/solr/cli/HealthcheckToolTest.java | 8 +- .../org/apache/solr/cli/PackageToolTest.java | 74 ++--- .../SolrCLIZkToolsTest.java} | 297 +++++++++--------- .../solr/cli/TestSolrCLIRunExample.java | 22 +- .../apache/solr/cli/ZkSubcommandsTest.java | 64 ++-- .../solr/cloud/SolrCloudExampleTest.java | 26 +- .../security/BasicAuthIntegrationTest.java | 2 +- solr/example/README.md | 2 +- solr/packaging/README.txt | 4 +- solr/packaging/test/bats_helper.bash | 2 +- solr/packaging/test/test_assert.bats | 2 +- solr/packaging/test/test_auth.bats | 18 +- solr/packaging/test/test_basic_auth.bats | 22 +- solr/packaging/test/test_bats.bats | 4 +- solr/packaging/test/test_config.bats | 31 +- solr/packaging/test/test_create.bats | 2 +- .../test/test_create_collection.bats | 28 +- .../test/test_delete_collection.bats | 10 +- .../packaging/test/test_example_noprompt.bats | 6 +- solr/packaging/test/test_export.bats | 4 +- solr/packaging/test/test_extraction.bats | 2 +- solr/packaging/test/test_healthcheck.bats | 2 +- solr/packaging/test/test_help.bats | 30 +- solr/packaging/test/test_modules.bats | 4 +- solr/packaging/test/test_packages.bats | 4 +- .../packaging/test/test_placement_plugin.bats | 2 +- solr/packaging/test/test_post.bats | 94 +++--- solr/packaging/test/test_postlogs.bats | 2 +- .../packaging/test/test_security_manager.bats | 6 +- solr/packaging/test/test_ssl.bats | 60 ++-- solr/packaging/test/test_start_solr.bats | 6 +- solr/packaging/test/test_status.bats | 11 +- solr/packaging/test/test_zk.bats | 36 ++- solr/packaging/test/test_zz_cleanup.bats | 4 +- .../prometheus/exporter/SolrExporter.java | 2 +- solr/server/solr/zoo.cfg | 2 +- .../pages/configuring-solr-xml.adoc | 2 +- .../pages/package-manager.adoc | 2 +- .../deployment-guide/pages/enabling-ssl.adoc | 8 +- .../pages/installing-solr.adoc | 2 +- ...onitoring-with-prometheus-and-grafana.adoc | 2 +- .../deployment-guide/pages/security-ui.adoc | 2 +- .../pages/solr-control-script-reference.adoc | 196 ++++++------ .../pages/solr-in-docker.adoc | 2 +- .../pages/zookeeper-file-management.adoc | 2 +- .../pages/zookeeper-utilities.adoc | 2 +- .../getting-started/pages/solr-tutorial.adoc | 2 +- .../getting-started/pages/tutorial-films.adoc | 2 +- .../pages/tutorial-five-minutes.adoc | 2 +- .../pages/tutorial-solrcloud.adoc | 12 +- .../pages/tutorial-techproducts.adoc | 8 +- .../indexing-guide/pages/post-tool.adoc | 95 +++--- .../indexing-guide/pages/schemaless-mode.adoc | 2 +- .../modules/query-guide/pages/logs.adoc | 2 +- .../pages/major-changes-in-solr-10.adoc | 5 + .../pages/solr-upgrade-notes.adoc | 1 - .../test-files/solrj/solr/multicore/zoo.cfg | 2 +- .../solr/cloud/AbstractDistribZkTestBase.java | 11 +- solr/webapp/web/partials/security.html | 2 +- 92 files changed, 1367 insertions(+), 1062 deletions(-) rename solr/core/src/test/org/apache/solr/{cloud/SolrCLIZkUtilsTest.java => cli/SolrCLIZkToolsTest.java} (86%) diff --git a/dev-docs/FAQ.adoc b/dev-docs/FAQ.adoc index 230fd0f1e41..b25d81bd6a9 100644 --- a/dev-docs/FAQ.adoc +++ b/dev-docs/FAQ.adoc @@ -26,7 +26,7 @@ You can review instructions for running Solr in Docker in the xref:running-in-do === Whats the fastest build lifecycle for frontend work on Solr Admin? Run `gradle dev`, and then `cd ./packaging/build/dev/`. Fire up your cluster -via `bin/solr start -e cloud -noprompt` and then as you make changes to assets in `/solr/webapp/web`, +via `bin/solr start -e cloud --no-prompt` and then as you make changes to assets in `/solr/webapp/web`, run `gradle dev` to redeploy the web assets. Do a hard refresh in your browser to pick up your changes. @@ -97,4 +97,3 @@ If you don't yet have an account, you have to ask for one in the 'users' or 'dev * http://fucit.org/solr-jenkins-reports/failure-report.html * https://ge.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr* * https://lists.apache.org[Solr mailing list archives especially builds] - diff --git a/dev-tools/scripts/cloud.sh b/dev-tools/scripts/cloud.sh index 6f4f4bb548b..3ab979d5ac3 100755 --- a/dev-tools/scripts/cloud.sh +++ b/dev-tools/scripts/cloud.sh @@ -352,7 +352,7 @@ stop() { SOLR=${CLUSTER_WD}/$(find . -maxdepth 1 -name 'solr*' -type d -print0 | xargs -0 ls -1 -td | sed -E 's/\.\/(solr.*)/\1/' | head -n1) popd - "${SOLR}/bin/solr" stop -all + "${SOLR}/bin/solr" stop --all } ######################## diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6c89c9867ee..400a7925684 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -135,6 +135,10 @@ Improvements * SOLR-15591: Make using debugger in Solr easier by avoiding NPE in ExternalPaths.determineSourceHome. (@charlygrappa via Eric Pugh) +* SOLR-16824: Adopt Linux standard pattern of -- for long option commands, and make all commands "kebab" formatting. I.e -zkHost is now -zk-host. The old parameters + such as -zkHost continue to be supported in the 9.x line of Solr. -u is now used to specify user credentials everywhere, this only impacts the bin/solr assert + commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski) + Optimizations --------------------- * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather diff --git a/solr/bin/solr b/solr/bin/solr index a2b9445604c..ac6c4330b79 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -17,7 +17,7 @@ # CONTROLLING STARTUP: # -# Use solr -help to see available command-line options. In addition +# Use solr --help to see available command-line options. In addition # to passing command-line options, this script looks for an include # file named solr.in.sh to set environment variables. Specifically, # the following locations are searched in this order: @@ -367,30 +367,30 @@ function print_usage() { if [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then echo "" - echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]" + echo "Usage: solr $CMD [-f] [-c] [--host host] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]" echo "" echo " -f Start Solr in foreground; default starts Solr in the background" echo " and sends stdout / stderr to solr-PORT-console.log" echo "" - echo " -c or -cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in" + echo " -c or --cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in" echo " solr.in.sh, an embedded ZooKeeper instance is started on Solr port+1000," echo " such as 9983 if Solr is bound to 8983" echo "" - echo " -host Specify the hostname for this Solr instance" + echo " --host Specify the hostname for this Solr instance" echo "" - echo " -p Specify the port to start the Solr HTTP listener on; default is 8983" + echo " -p/--port Specify the port to start the Solr HTTP listener on; default is 8983" echo " The specified port (SOLR_PORT) will also be used to determine the stop port" echo " STOP_PORT=(\$SOLR_PORT-1000) and JMX RMI listen port RMI_PORT=(\$SOLR_PORT+10000). " echo " For instance, if you set -p 8985, then the STOP_PORT=7985 and RMI_PORT=18985" echo "" echo " -d Specify the Solr server directory; defaults to server" echo "" - echo " -z/-zkHost Zookeeper connection string; only used when running in SolrCloud mode using -c" + echo " -z/--zk-host Zookeeper connection string; only used when running in SolrCloud mode using -c" echo " If neither ZK_HOST is defined in solr.in.sh nor the -z parameter is specified," echo " an embedded ZooKeeper instance will be launched." echo " Set the ZK_CREATE_CHROOT environment variable to true if your ZK host has a chroot path, and you want to create it automatically." echo "" - echo " -m Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g" + echo " -m/--memory Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g" echo " results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m" echo "" echo " -s Sets the solr.solr.home system property; Solr will create core directories under" @@ -401,7 +401,7 @@ function print_usage() { echo " on which example is run. The default value is server/solr. If passed relative dir," echo " validation with current dir will be done, before trying default server/" echo "" - echo " -t Sets the solr.data.home system property, where Solr will store index data in /data subdirectories." + echo " -t/--data-home Sets the solr.data.home system property, where Solr will store index data in /data subdirectories." echo " If not set, Solr uses solr.solr.home for config and data." echo "" echo " -e Name of the example to run; available examples:" @@ -420,14 +420,14 @@ function print_usage() { echo " you could pass: -j \"--include-jetty-dir=/etc/jetty/custom/server/\"" echo " In most cases, you should wrap the additional parameters in double quotes." echo "" - echo " -noprompt Don't prompt for input; accept all defaults when running examples that accept user input" + echo " --no-prompt Don't prompt for input; accept all defaults when running examples that accept user input" echo "" - echo " -force If attempting to start Solr as the root user, the script will exit with a warning that running Solr as \"root\" can cause problems." - echo " It is possible to override this warning with the '-force' parameter." + echo " --force If attempting to start Solr as the root user, the script will exit with a warning that running Solr as \"root\" can cause problems." + echo " It is possible to override this warning with the '--force' parameter." echo "" echo " -v and -q Verbose (-v) or quiet (-q) logging. Sets default log level of Solr to DEBUG or WARN instead of INFO" echo "" - echo " -V/-verbose Verbose messages from this script" + echo " -V/--verbose Verbose messages from this script" echo "" elif [ "$CMD" == "stop" ]; then echo "" @@ -437,9 +437,9 @@ function print_usage() { echo "" echo " -p Specify the port the Solr HTTP listener is bound to" echo "" - echo " -all Find and stop all running Solr servers on this host" + echo " --all Find and stop all running Solr servers on this host" echo "" - echo " -V/-verbose Verbose messages from this script" + echo " -V/--verbose Verbose messages from this script" echo "" echo " NOTE: To see if any Solr servers are running, do: solr status" echo "" @@ -453,7 +453,7 @@ function print_usage() { echo "" echo " -s solrUrl Optional Solr URL to look up the correct zkHost connection string via." echo "" - echo " -V/-verbose Enable more verbose output for this script." + echo " -v/--verbose Enable more verbose output for this script." echo "" echo " upconfig uploads a configset from the local machine to Zookeeper." echo "" @@ -528,21 +528,21 @@ function print_usage() { echo "" elif [ "$CMD" == "auth" ]; then echo "" - echo "Usage: solr auth enable [-type basicAuth] -credentials user:pass [-blockUnknown ] [-updateIncludeFileOnly ] [-V]" - echo " solr auth enable [-type basicAuth] -prompt [-blockUnknown ] [-updateIncludeFileOnly ] [-V]" - echo " solr auth enable -type kerberos -config \"\" [-updateIncludeFileOnly ] [-V]" - echo " solr auth disable [-updateIncludeFileOnly ] [-V]" + echo "Usage: solr auth enable [--type basicAuth] --credentials user:pass [--block-unknown ] [--update-include-file-only ] [-v]" + echo " solr auth enable [--type basicAuth] --prompt [--block-unknown ] [--update-include-file-only ] [-v]" + echo " solr auth enable --type kerberos --config \"\" [--update-include-file-only ] [-v]" + echo " solr auth disable [--update-include-file-only ] [-v]" echo "" echo " Updates or enables/disables authentication. Must be run on the machine hosting Solr." echo "" - echo " -type or -t The authentication mechanism (basicAuth or kerberos) to enable. Defaults to 'basicAuth'." + echo " --type or -t The authentication mechanism (basicAuth or kerberos) to enable. Defaults to 'basicAuth'." echo "" - echo " -credentials The username and password of the initial user. Applicable for basicAuth only." - echo " Note: only one of -prompt or -credentials must be provided" + echo " --credentials The username and password of the initial user. Applicable for basicAuth only." + echo " Note: only one of --prompt or --credentials must be provided" echo "" - echo " -config \"\" Configuration parameters (Solr startup parameters). Required and applicable only for Kerberos" + echo " --config \"\" Configuration parameters (Solr startup parameters). Required and applicable only for Kerberos" echo "" - echo " -solrIncludeFile Specify the full path to the include file in the environment." + echo " --solr-include-file Specify the full path to the include file in the environment." echo " If not specified this script looks for an include file named solr.in.sh to set environment variables. " echo " Specifically,the following locations are searched in this order:" echo " From 2b05db2997f13efbdf1b08983df690eef626e215 Mon Sep 17 00:00:00 2001 From: Yuntong Qu Date: Mon, 15 Jul 2024 11:51:29 -0400 Subject: [PATCH 037/172] ref doc format fix (#2555) missing a + --- .../modules/deployment-guide/pages/collection-management.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index 8c6b6e2bc76..295392f9d2c 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -1648,6 +1648,7 @@ This parameter has no effect if `incremental=false` is specified. Indicates if configset files should be included with the index backup or not. Note that in order to restore a collection, the configset must either exist in ZooKeeper or be part of the backup. Only set this to `false` if you can restore configsets by other means external to Solr (i.e. you have it stored with your application source code, is part of your ZooKeeper backups, etc). `property.` (V1), `extraProperties` (V2):: ++ [%autowidth,frame=none] |=== |Optional |Default: none From bd3d2f0d4e47fff4db4d1221c1540f15d03ba579 Mon Sep 17 00:00:00 2001 From: Yuntong Qu Date: Mon, 15 Jul 2024 11:52:53 -0400 Subject: [PATCH 038/172] [ref guide] Fix description of field backupName for backup (#2554) the previous description of backupName is not correct. Backup name should be the same if using incremental backups --- .../modules/deployment-guide/pages/collection-management.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index 295392f9d2c..62634d79ea0 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -1588,7 +1588,8 @@ s|Required |Default: none + What to name the backup that is created. Provided as a query parameter for v1 requests, or as a path segment for v2 requests. -This is checked to make sure it doesn't already exist, and otherwise an error message is raised. ++ +For incremental backups, the backup name should be reused to add new backup points to the existing backup. For non-incremental backups (deprecated), this name is checked to ensure it doesn't already exist, and an error message is raised if it does. `location`:: + From d3b4c2e1ae39b8ecc5428798531f8b7cf723d787 Mon Sep 17 00:00:00 2001 From: Pierre Salagnac <82967811+psalagnac@users.noreply.github.com> Date: Mon, 15 Jul 2024 23:04:36 +0200 Subject: [PATCH 039/172] SOLR-17160: Core admin async ID status, 10k limit and time expire (#2304) Core Admin "async" request status tracking is no longer capped at 100; it's 10k. Statuses are now removed 5 minutes after the read of a completed/failed status. Helps collection async backup/restore and other operations scale to 100+ shards. Co-authored-by: David Smiley --- solr/CHANGES.txt | 4 + .../solr/handler/admin/CoreAdminHandler.java | 153 +++++++++++++----- .../admin/api/GetNodeCommandStatus.java | 33 ++-- .../handler/admin/CoreAdminHandlerTest.java | 66 ++++++++ .../admin/api/GetNodeCommandStatusTest.java | 9 +- .../org/apache/solr/common/util/EnvUtils.java | 14 ++ 6 files changed, 214 insertions(+), 65 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c7ec6e000c8..38231465f4d 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -150,6 +150,10 @@ Improvements * SOLR-16198: Introduce tabbed sections again in the Ref Guide. (Christos Malliaridis via Eric Pugh) +* SOLR-17160: Core Admin "async" request status tracking is no longer capped at 100; it's 10k. + Statuses are now removed 5 minutes after the read of a completed/failed status. Helps collection + async backup/restore and other operations scale to 100+ shards. (Pierre Salagnac, David Smiley) + Optimizations --------------------- * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index d2d823eb190..c35117fb088 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -21,20 +21,23 @@ import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import com.github.benmanes.caffeine.cache.Ticker; import java.io.File; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.solr.api.AnnotatedApi; import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; @@ -47,6 +50,7 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; @@ -427,11 +431,19 @@ default boolean isExpensive() { } public static class CoreAdminAsyncTracker { - private static final int MAX_TRACKED_REQUESTS = 100; + /** + * Max number of requests we track in the Caffeine cache. This limit is super high on purpose, + * we're not supposed to hit it. This is just a protection to grow in memory too much when + * receiving an abusive number of admin requests. + */ + private static final int MAX_TRACKED_REQUESTS = + EnvUtils.getPropertyAsInteger("solr.admin.async.max", 10_000); + public static final String RUNNING = "running"; public static final String COMPLETED = "completed"; public static final String FAILED = "failed"; - public final Map> requestStatusMap; + + private final Cache requestStatusCache; // key by ID // Executor for all standard tasks (the ones that are not flagged as expensive) // We always keep 50 live threads @@ -440,7 +452,7 @@ public static class CoreAdminAsyncTracker { 50, new SolrNamedThreadFactory("parallelCoreAdminAPIBaseExecutor")); // Executor for expensive tasks - // We keep the number number of max threads very low to have throttling for expensive tasks + // We keep the number of max threads very low to have throttling for expensive tasks private ExecutorService expensiveExecutor = ExecutorUtil.newMDCAwareCachedThreadPool( 5, @@ -448,11 +460,28 @@ public static class CoreAdminAsyncTracker { new SolrNamedThreadFactory("parallelCoreAdminAPIExpensiveExecutor")); public CoreAdminAsyncTracker() { - HashMap> map = new HashMap<>(3, 1.0f); - map.put(RUNNING, Collections.synchronizedMap(new LinkedHashMap<>())); - map.put(COMPLETED, Collections.synchronizedMap(new LinkedHashMap<>())); - map.put(FAILED, Collections.synchronizedMap(new LinkedHashMap<>())); - requestStatusMap = Collections.unmodifiableMap(map); + this( + Ticker.systemTicker(), + TimeUnit.MINUTES.toNanos( + EnvUtils.getPropertyAsLong("solr.admin.async.timeout.minutes", 60L)), + TimeUnit.MINUTES.toNanos( + EnvUtils.getPropertyAsLong("solr.admin.async.timeout.completed.minutes", 5L))); + } + + /** + * @param runningTimeoutNanos The time-to-keep for tasks in the RUNNING state. + * @param completedTimeoutNanos The time-to-keep for tasks in the COMPLETED or FAILED state + * after the status was polled. + */ + CoreAdminAsyncTracker(Ticker ticker, long runningTimeoutNanos, long completedTimeoutNanos) { + + TaskExpiry expiry = new TaskExpiry(runningTimeoutNanos, completedTimeoutNanos); + requestStatusCache = + Caffeine.newBuilder() + .ticker(ticker) + .maximumSize(MAX_TRACKED_REQUESTS) + .expireAfter(expiry) + .build(); } public void shutdown() { @@ -460,13 +489,22 @@ public void shutdown() { ExecutorUtil.shutdownAndAwaitTermination(expensiveExecutor); } - public Map getRequestStatusMap(String key) { - return requestStatusMap.get(key); + public TaskObject getAsyncRequestForStatus(String key) { + TaskObject task = requestStatusCache.getIfPresent(key); + + if (task != null && !RUNNING.equals(task.status) && !task.polledAfterCompletion) { + task.polledAfterCompletion = true; + // At the first time we retrieve the status of a completed request, do a second lookup in + // the cache. This is necessary to update the TTL of this request in the cache. + // Unfortunately, we can't force the expiration time to be refreshed without a lookup. + requestStatusCache.getIfPresent(key); + } + + return task; } public void submitAsyncTask(TaskObject taskObject) throws SolrException { - ensureTaskIdNotInUse(taskObject.taskId); - addTask(RUNNING, taskObject); + addTask(taskObject); Runnable command = () -> { @@ -497,42 +535,26 @@ public void submitAsyncTask(TaskObject taskObject) throws SolrException { } } - /** Helper method to add a task to a tracking type. */ - private void addTask(String type, TaskObject o, boolean limit) { - synchronized (getRequestStatusMap(type)) { - if (limit && getRequestStatusMap(type).size() == MAX_TRACKED_REQUESTS) { - String key = getRequestStatusMap(type).entrySet().iterator().next().getKey(); - getRequestStatusMap(type).remove(key); - } - addTask(type, o); - } - } - - private void addTask(String type, TaskObject o) { - synchronized (getRequestStatusMap(type)) { - getRequestStatusMap(type).put(o.taskId, o); - } - } - - /** Helper method to remove a task from a tracking map. */ - private void removeTask(String map, String taskId) { - synchronized (getRequestStatusMap(map)) { - getRequestStatusMap(map).remove(taskId); - } - } - - private void ensureTaskIdNotInUse(String taskId) throws SolrException { - if (getRequestStatusMap(RUNNING).containsKey(taskId) - || getRequestStatusMap(COMPLETED).containsKey(taskId) - || getRequestStatusMap(FAILED).containsKey(taskId)) { + private void addTask(TaskObject taskObject) { + // Ensure task ID is not already in use + TaskObject taskInCache = + requestStatusCache.get( + taskObject.taskId, + n -> { + taskObject.status = RUNNING; + return taskObject; + }); + + // If we get a different task instance, it means one was already in the cache with the + // same name. Just reject the new one. + if (taskInCache != taskObject) { throw new SolrException( ErrorCode.BAD_REQUEST, "Duplicate request with the same requestid found."); } } private void finishTask(TaskObject taskObject, boolean successful) { - removeTask(RUNNING, taskObject.taskId); - addTask(successful ? COMPLETED : FAILED, taskObject, true); + taskObject.status = successful ? COMPLETED : FAILED; } /** @@ -546,6 +568,13 @@ public static class TaskObject { final Callable task; public String rspInfo; public Object operationRspInfo; + private volatile String status; + + /** + * Flag set to true once the task is complete (can be in error) and the status was polled + * already once. Once set, the time we keep the task status is shortened. + */ + private volatile boolean polledAfterCompletion; public TaskObject( String taskId, String action, boolean expensive, Callable task) { @@ -574,6 +603,42 @@ public Object getOperationRspObject() { public void setOperationRspObject(SolrQueryResponse rspObject) { this.operationRspInfo = rspObject.getResponse(); } + + public String getStatus() { + return status; + } + } + + /** + * Expiration policy for Caffeine cache. Depending on whether the status of a completed task was + * already retrieved, we return {@link #runningTimeoutNanos} or {@link #completedTimeoutNanos}. + */ + private static class TaskExpiry implements Expiry { + + private final long runningTimeoutNanos; + private final long completedTimeoutNanos; + + private TaskExpiry(long runningTimeoutNanos, long completedTimeoutNanos) { + this.runningTimeoutNanos = runningTimeoutNanos; + this.completedTimeoutNanos = completedTimeoutNanos; + } + + @Override + public long expireAfterCreate(String key, TaskObject task, long currentTime) { + return runningTimeoutNanos; + } + + @Override + public long expireAfterUpdate( + String key, TaskObject task, long currentTime, long currentDuration) { + return task.polledAfterCompletion ? completedTimeoutNanos : runningTimeoutNanos; + } + + @Override + public long expireAfterRead( + String key, TaskObject task, long currentTime, long currentDuration) { + return task.polledAfterCompletion ? completedTimeoutNanos : runningTimeoutNanos; + } } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetNodeCommandStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetNodeCommandStatus.java index c6627d012e3..258e83e16a0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetNodeCommandStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetNodeCommandStatus.java @@ -18,7 +18,6 @@ import static org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.COMPLETED; import static org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.FAILED; -import static org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.RUNNING; import jakarta.inject.Inject; import org.apache.solr.client.api.endpoint.GetNodeCommandStatusApi; @@ -26,6 +25,7 @@ import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.TaskObject; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -51,25 +51,24 @@ public GetNodeCommandStatus( public GetNodeCommandStatusResponse getCommandStatus(String requestId) { ensureRequiredParameterProvided(CoreAdminParams.REQUESTID, requestId); var requestStatusResponse = new GetNodeCommandStatusResponse(); - if (coreAdminAsyncTracker.getRequestStatusMap(RUNNING).containsKey(requestId)) { - requestStatusResponse.status = RUNNING; - } else if (coreAdminAsyncTracker.getRequestStatusMap(COMPLETED).containsKey(requestId)) { - requestStatusResponse.status = COMPLETED; - requestStatusResponse.response = - coreAdminAsyncTracker.getRequestStatusMap(COMPLETED).get(requestId).getRspObject(); - requestStatusResponse.response = - coreAdminAsyncTracker - .getRequestStatusMap(COMPLETED) - .get(requestId) - .getOperationRspObject(); - } else if (coreAdminAsyncTracker.getRequestStatusMap(FAILED).containsKey(requestId)) { - requestStatusResponse.status = FAILED; - requestStatusResponse.response = - coreAdminAsyncTracker.getRequestStatusMap(FAILED).get(requestId).getRspObject(); - } else { + + TaskObject taskObject = coreAdminAsyncTracker.getAsyncRequestForStatus(requestId); + String status = taskObject != null ? taskObject.getStatus() : null; + + if (status == null) { requestStatusResponse.status = "notfound"; requestStatusResponse.msg = "No task found in running, completed or failed tasks"; + } else { + requestStatusResponse.status = status; + + if (status.equals(COMPLETED)) { + requestStatusResponse.response = taskObject.getRspObject(); + requestStatusResponse.response = taskObject.getOperationRspObject(); + } else if (status.equals(FAILED)) { + requestStatusResponse.response = taskObject.getRspObject(); + } } + return requestStatusResponse; } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index 8a1bafde030..2dd9fce1224 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.handler.admin; +import static org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.COMPLETED; + import java.io.Reader; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -24,7 +26,9 @@ import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.file.PathUtils; import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; @@ -44,6 +48,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker; +import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.TaskObject; import org.apache.solr.response.SolrQueryResponse; import org.junit.BeforeClass; import org.junit.Test; @@ -532,4 +538,64 @@ public void testNonexistentCoreReload() throws Exception { e.getMessage()); admin.close(); } + + @Test + public void testTrackedRequestExpiration() throws Exception { + // Create a tracker with controlled clock, relative to 0 + AtomicLong clock = new AtomicLong(0L); + CoreAdminAsyncTracker asyncTracker = new CoreAdminAsyncTracker(clock::get, 100L, 10L); + try { + Set tasks = + Set.of( + new TaskObject("id1", "ACTION", false, SolrQueryResponse::new), + new TaskObject("id2", "ACTION", false, SolrQueryResponse::new)); + + // Submit all tasks and wait for internal status to be COMPLETED + tasks.forEach(asyncTracker::submitAsyncTask); + while (!tasks.stream().allMatch(t -> COMPLETED.equals(t.getStatus()))) { + Thread.sleep(10L); + } + + // Timeout for running tasks is 100n, so status can be retrieved after 20n. + // But timeout for complete tasks is 10n once we polled the status at least once, so status + // is not available anymore 20n later. + clock.set(20); + assertEquals(COMPLETED, asyncTracker.getAsyncRequestForStatus("id1").getStatus()); + clock.set(40L); + assertNull(asyncTracker.getAsyncRequestForStatus("id1")); + + // Move the clock after the running timeout. + // Status of second task is not available anymore, even if it wasn't retrieved yet + clock.set(110L); + assertNull(asyncTracker.getAsyncRequestForStatus("id2")); + + } finally { + asyncTracker.shutdown(); + } + } + + /** Check we reject a task is the async ID already exists. */ + @Test + public void testDuplicatedRequestId() { + + // Different tasks but with same ID + TaskObject task1 = new TaskObject("id1", "ACTION", false, null); + TaskObject task2 = new TaskObject("id1", "ACTION", false, null); + + CoreAdminAsyncTracker asyncTracker = new CoreAdminAsyncTracker(); + try { + asyncTracker.submitAsyncTask(task1); + try { + asyncTracker.submitAsyncTask(task2); + fail("Task should have been rejected."); + } catch (SolrException e) { + assertEquals("Duplicate request with the same requestid found.", e.getMessage()); + } + + assertNotNull(task1.getStatus()); + assertNull(task2.getStatus()); + } finally { + asyncTracker.shutdown(); + } + } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/GetNodeCommandStatusTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/GetNodeCommandStatusTest.java index 4a52383bf20..b22d3d4d4ec 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/GetNodeCommandStatusTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/GetNodeCommandStatusTest.java @@ -19,12 +19,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.Map; import org.apache.solr.SolrTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.endpoint.GetNodeCommandStatusApi; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminAsyncTracker.TaskObject; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -33,7 +33,6 @@ public class GetNodeCommandStatusTest extends SolrTestCase { private CoreContainer mockCoreContainer; private CoreAdminHandler.CoreAdminAsyncTracker mockAsyncTracker; - private CoreAdminHandler.CoreAdminAsyncTracker.TaskObject taskObject; private GetNodeCommandStatusApi requestNodeCommandApi; @BeforeClass @@ -45,7 +44,6 @@ public static void ensureWorkingMockito() { public void setupMocks() { mockCoreContainer = mock(CoreContainer.class); mockAsyncTracker = mock(CoreAdminHandler.CoreAdminAsyncTracker.class); - taskObject = new CoreAdminHandler.CoreAdminAsyncTracker.TaskObject(null, null, false, null); requestNodeCommandApi = new GetNodeCommandStatus(mockCoreContainer, mockAsyncTracker, null, null); } @@ -89,6 +87,9 @@ public void testReturnsStatusOfFailedCommandId() { } private void whenTaskExistsWithStatus(String taskId, String status) { - when(mockAsyncTracker.getRequestStatusMap(status)).thenReturn(Map.of(taskId, taskObject)); + TaskObject taskObject = mock(TaskObject.class); + when(taskObject.getStatus()).thenReturn(status); + + when(mockAsyncTracker.getAsyncRequestForStatus(taskId)).thenReturn(taskObject); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/EnvUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/EnvUtils.java index 40bb7c0fdb3..6adac222ab7 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/EnvUtils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/EnvUtils.java @@ -111,6 +111,20 @@ private static String camelCaseToDotSeparated(String key) { } /** Get property as integer */ + public static Integer getPropertyAsInteger(String key) { + return getPropertyAsInteger(key, null); + } + + /** Get property as integer, or default value */ + public static Integer getPropertyAsInteger(String key, Integer defaultValue) { + String value = getProperty(key); + if (value == null) { + return defaultValue; + } + return Integer.parseInt(value); + } + + /** Get property as long */ public static Long getPropertyAsLong(String key) { return getPropertyAsLong(key, null); } From cdb7af363afbaea6955013a7d8087f447f8dca20 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 16 Jul 2024 17:02:25 +0200 Subject: [PATCH 040/172] Review org.apache.solr.client.solrj.io.stream and responding to some warnings. (#2530) Review org.apache.solr.client.solrj.io.stream and responding to some intellij warnings. Typos, methods not used, return values not used, that sort of thing.. --------- Co-authored-by: Christine Poerschke --- .../solrj/io/stream/CloudAuthStreamTest.java | 20 +++--- .../solrj/io/stream/JDBCStreamTest.java | 45 +++---------- .../ParallelFacetStreamOverAliasTest.java | 4 +- .../io/stream/SelectWithEvaluatorsTest.java | 66 ++----------------- .../solrj/io/stream/StreamDecoratorTest.java | 14 ++-- .../io/stream/StreamExecutorHelperTest.java | 2 +- .../solrj/io/stream/StreamExpressionTest.java | 23 ++----- ... => StreamExpressionToExpressionTest.java} | 9 ++- .../client/solrj/io/stream/StreamingTest.java | 3 +- 9 files changed, 43 insertions(+), 143 deletions(-) rename solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/{StreamExpressionToExpessionTest.java => StreamExpressionToExpressionTest.java} (98%) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/CloudAuthStreamTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/CloudAuthStreamTest.java index cc860697c42..799330fbd0e 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/CloudAuthStreamTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/CloudAuthStreamTest.java @@ -50,7 +50,7 @@ /** * tests various streaming expressions (via the SolrJ {@link SolrStream} API) against a SolrCloud - * cluster using both Authenticationand Role based Authorization + * cluster using both Authentication and Role based Authorization */ public class CloudAuthStreamTest extends SolrCloudTestCase { @@ -67,8 +67,8 @@ public class CloudAuthStreamTest extends SolrCloudTestCase { private static String solrUrl = null; /** - * Helper that returns the original {@link SolrRequest} with it's original type so it can - * be chained. This menthod knows that for the purpose of this test, every user name is it's own + * Helper that returns the original {@link SolrRequest} with its original type so it can + * be chained. This method knows that for the purpose of this test, every username is its own * password * * @see SolrRequest#setBasicAuthCredentials @@ -253,7 +253,7 @@ public void testEchoStream() throws Exception { assertEquals("hello world", tuples.get(0).get("echo")); } - public void testEchoStreamNoCredentials() throws Exception { + public void testEchoStreamNoCredentials() { final SolrStream solrStream = new SolrStream( solrUrl + "/" + COLLECTION_X, @@ -270,7 +270,7 @@ public void testEchoStreamNoCredentials() throws Exception { }); } - public void testEchoStreamInvalidCredentials() throws Exception { + public void testEchoStreamInvalidCredentials() { final SolrStream solrStream = new SolrStream( solrUrl + "/" + COLLECTION_X, @@ -489,7 +489,7 @@ public void testExecutorUpdateStreamInsufficientCredentials() throws Exception { params("qt", "/stream", "_trace", "executor_via_" + trace, "expr", expr)); solrStream.setCredentials(user, user); - // NOTE: Becaue of the backgroun threads, no failures will to be returned to client... + // NOTE: Because of the background threads, no failures will to be returned to client... final List tuples = getTuples(solrStream); assertEquals(0, tuples.size()); @@ -511,7 +511,7 @@ public void testDaemonUpdateStream() throws Exception { { // NOTE: in spite of what is implied by 'terminate=true', this daemon will NEVER terminate on - // it's own as long as the updates are successful (apparently that requires usage of a topic() + // its own as long as the updates are successful (apparently that requires usage of a topic() // stream to set a "sleepMillis"?!) final String expr = "daemon(id=daemonId,runInterval=1000,terminate=true,update(" @@ -936,7 +936,7 @@ protected static long countDocsInCollection(final String collection, final Strin /** Slurps a stream into a List */ protected static List getTuples(final TupleStream tupleStream) throws IOException { - List tuples = new ArrayList(); + List tuples = new ArrayList<>(); try { log.trace("TupleStream: {}", tupleStream); tupleStream.open(); @@ -952,8 +952,8 @@ protected static List getTuples(final TupleStream tupleStream) throws IOE return tuples; } - /** Sigh. DaemonStream requires polling the same core where the stream was exectured. */ - protected static String getRandomCoreUrl(final String collection) throws Exception { + /** Sigh. DaemonStream requires polling the same core where the stream was executed. */ + protected static String getRandomCoreUrl(final String collection) { final List replicaUrls = cluster .getZkStateReader() diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java index 635376be1df..3a2e028130d 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java @@ -25,7 +25,6 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Map; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressPointFields; @@ -94,7 +93,7 @@ public static void setupCluster() throws Exception { public static void setupDatabase() throws Exception { // Initialize Database - // Ok, so.....hsqldb is doing something totally weird so I thought I'd take a moment to explain + // Ok, so.....hsqldb is doing something totally weird, so I thought I'd take a moment to explain // it. // According to http://www.hsqldb.org/doc/1.8/guide/guide.html#N101EF, section "Components of // SQL Expressions", clause "name", "When an SQL statement is issued, any lowercase characters @@ -279,7 +278,7 @@ public void testJDBCSolrMerge() throws Exception { // Load Solr new UpdateRequest() - .add(id, "0", "code_s", "GB", "name_s", "Great Britian") + .add(id, "0", "code_s", "GB", "name_s", "Great Britain") .add(id, "1", "code_s", "CA", "name_s", "Canada") .commit(cluster.getSolrClient(), COLLECTIONORALIAS); @@ -323,7 +322,7 @@ public void testJDBCSolrMerge() throws Exception { "name_s", "Algeria", "Canada", - "Great Britian", + "Great Britain", "Netherlands", "Norway", "Nepal", @@ -773,7 +772,7 @@ protected List getTuples(TupleStream tupleStream) throws IOException { return tuples; } - protected boolean assertOrderOf(List tuples, String fieldName, int... values) + protected void assertOrderOf(List tuples, String fieldName, int... values) throws Exception { int i = 0; for (int val : values) { @@ -784,11 +783,9 @@ protected boolean assertOrderOf(List tuples, String fieldName, int... val } ++i; } - return true; } - protected boolean assertOrderOf(List tuples, String fieldName, double... values) - throws Exception { + protected void assertOrderOf(List tuples, String fieldName, double... values) { int i = 0; for (double val : values) { Tuple t = tuples.get(i); @@ -796,10 +793,9 @@ protected boolean assertOrderOf(List tuples, String fieldName, double... assertEquals("Found value:" + tip + " expecting:" + val, val, tip, 0.00001); ++i; } - return true; } - protected boolean assertOrderOf(List tuples, String fieldName, String... values) + protected void assertOrderOf(List tuples, String fieldName, String... values) throws Exception { int i = 0; for (String val : values) { @@ -807,7 +803,7 @@ protected boolean assertOrderOf(List tuples, String fieldName, String... if (null == val) { if (null != t.get(fieldName)) { - throw new Exception("Found value:" + (String) t.get(fieldName) + " expecting:null"); + throw new Exception("Found value:" + t.get(fieldName) + " expecting:null"); } } else { String tip = (String) t.get(fieldName); @@ -817,29 +813,6 @@ protected boolean assertOrderOf(List tuples, String fieldName, String... } ++i; } - return true; - } - - protected boolean assertFields(List tuples, String... fields) throws Exception { - for (Tuple tuple : tuples) { - for (String field : fields) { - if (!tuple.getFields().containsKey(field)) { - throw new Exception(String.format(Locale.ROOT, "Expected field '%s' not found", field)); - } - } - } - return true; - } - - protected boolean assertNotFields(List tuples, String... fields) throws Exception { - for (Tuple tuple : tuples) { - for (String field : fields) { - if (tuple.getFields().containsKey(field)) { - throw new Exception(String.format(Locale.ROOT, "Unexpected field '%s' found", field)); - } - } - } - return true; } public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception { @@ -862,9 +835,7 @@ public boolean assertDouble(Tuple tuple, String fieldName, double d) throws Exce public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception { String actual = (String) tuple.get(fieldName); - if ((null == expected && null != actual) - || (null != expected && null == actual) - || (null != expected && !expected.equals(actual))) { + if ((null == expected && null != actual) || (null != expected && !expected.equals(actual))) { throw new Exception("Longs not equal:" + expected + " : " + actual); } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/ParallelFacetStreamOverAliasTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/ParallelFacetStreamOverAliasTest.java index d7e90a09c42..347894bae42 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/ParallelFacetStreamOverAliasTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/ParallelFacetStreamOverAliasTest.java @@ -100,7 +100,7 @@ public static void setupCluster() throws Exception { solrClientCache = new SolrClientCache(); } - /** setup the testbed with necessary collections, documents, and alias */ + /** set up the testbed with necessary collections, documents, and alias */ public static void setupCollectionsAndAlias() throws Exception { final NormalDistribution[] dists = new NormalDistribution[CARDINALITY]; @@ -299,7 +299,7 @@ public void testParallelStats() throws Exception { assertNull(statsStream.parallelizedStream); } - // execute the provided expression with tiered=true and compare to results of tiered=false + // execute the provided expression with tiered=true and compare to result of tiered=false private void compareTieredStreamWithNonTiered(String facetExprTmpl, int dims) throws IOException { String facetExpr = String.format(Locale.US, facetExprTmpl, ALIAS_NAME, "true"); diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java index c2639919cb0..ac4a4102e55 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Map; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -47,8 +46,6 @@ public class SelectWithEvaluatorsTest extends SolrCloudTestCase { private static final int TIMEOUT = DEFAULT_TIMEOUT; private static final String id = "id"; - private static boolean useAlias; - @BeforeClass public static void setupCluster() throws Exception { configureCluster(4) @@ -71,7 +68,7 @@ public static void setupCluster() throws Exception { .configure(); String collection; - useAlias = random().nextBoolean(); + boolean useAlias = random().nextBoolean(); if (useAlias) { collection = COLLECTIONORALIAS + "_collection"; } else { @@ -145,39 +142,7 @@ protected List getTuples(TupleStream tupleStream) throws IOException { return tuples; } - protected boolean assertOrder(List tuples, int... ids) throws Exception { - return assertOrderOf(tuples, "id", ids); - } - - protected boolean assertOrderOf(List tuples, String fieldName, int... ids) - throws Exception { - int i = 0; - for (int val : ids) { - Tuple t = tuples.get(i); - String tip = t.getString(fieldName); - if (!tip.equals(Integer.toString(val))) { - throw new Exception("Found value:" + tip + " expecting:" + val); - } - ++i; - } - return true; - } - - protected boolean assertMapOrder(List tuples, int... ids) throws Exception { - int i = 0; - for (int val : ids) { - Tuple t = tuples.get(i); - List> tip = t.getMaps("group"); - int id = (int) tip.get(0).get("id"); - if (id != val) { - throw new Exception("Found value:" + id + " expecting:" + val); - } - ++i; - } - return true; - } - - protected boolean assertFields(List tuples, String... fields) throws Exception { + protected void assertFields(List tuples, String... fields) throws Exception { for (Tuple tuple : tuples) { for (String field : fields) { if (!tuple.getFields().containsKey(field)) { @@ -185,10 +150,9 @@ protected boolean assertFields(List tuples, String... fields) throws Exce } } } - return true; } - protected boolean assertNotFields(List tuples, String... fields) throws Exception { + protected void assertNotFields(List tuples, String... fields) throws Exception { for (Tuple tuple : tuples) { for (String field : fields) { if (tuple.getFields().containsKey(field)) { @@ -196,21 +160,6 @@ protected boolean assertNotFields(List tuples, String... fields) throws E } } } - return true; - } - - protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception { - List group = (List) tuple.get("tuples"); - int i = 0; - for (int val : ids) { - Map t = (Map) group.get(i); - Long tip = (Long) t.get("id"); - if (tip.intValue() != val) { - throw new Exception("Found value:" + tip.intValue() + " expecting:" + val); - } - ++i; - } - return true; } public boolean assertLong(Tuple tuple, String fieldName, long l) throws Exception { @@ -235,12 +184,9 @@ public boolean assertDouble(Tuple tuple, String fieldName, double expectedValue) public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception { String actual = (String) tuple.get(fieldName); - if ((null == expected && null != actual) - || (null != expected && null == actual) - || (null != expected && !expected.equals(actual))) { - throw new Exception("Longs not equal:" + expected + " : " + actual); + if ((null != expected || null == actual) && (null == expected || expected.equals(actual))) { + return true; } - - return true; + throw new Exception("Longs not equal:" + expected + " : " + actual); } } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 453528d8338..2b19742ae6f 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -5152,12 +5152,11 @@ protected List getTuples(TupleStream tupleStream) throws IOException { return tuples; } - protected boolean assertOrder(List tuples, int... ids) throws Exception { - return assertOrderOf(tuples, "id", ids); + protected void assertOrder(List tuples, int... ids) throws Exception { + assertOrderOf(tuples, "id", ids); } - protected boolean assertOrderOf(List tuples, String fieldName, int... ids) - throws Exception { + protected void assertOrderOf(List tuples, String fieldName, int... ids) throws Exception { int i = 0; for (int val : ids) { Tuple t = tuples.get(i); @@ -5167,10 +5166,9 @@ protected boolean assertOrderOf(List tuples, String fieldName, int... ids } ++i; } - return true; } - protected boolean assertFields(List tuples, String... fields) throws Exception { + protected void assertFields(List tuples, String... fields) throws Exception { for (Tuple tuple : tuples) { for (String field : fields) { if (!tuple.getFields().containsKey(field)) { @@ -5178,10 +5176,9 @@ protected boolean assertFields(List tuples, String... fields) throws Exce } } } - return true; } - protected boolean assertNotFields(List tuples, String... fields) throws Exception { + protected void assertNotFields(List tuples, String... fields) throws Exception { for (Tuple tuple : tuples) { for (String field : fields) { if (tuple.getFields().containsKey(field)) { @@ -5189,7 +5186,6 @@ protected boolean assertNotFields(List tuples, String... fields) throws E } } } - return true; } protected boolean assertGroupOrder(Tuple tuple, int... ids) throws Exception { diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExecutorHelperTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExecutorHelperTest.java index ae80cea1060..287aa851734 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExecutorHelperTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExecutorHelperTest.java @@ -36,7 +36,7 @@ public void submitAllTest() throws IOException { List results = new ArrayList<>(); results.addAll(StreamExecutorHelper.submitAllAndAwaitAggregatingExceptions(tasks, "test")); Collections.sort(results); - List expected = List.of(0l, 1l, 2l, 3l, 4l); + List expected = List.of(0L, 1L, 2L, 3L, 4L); assertEquals(expected, results); } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index 050b5726ff2..1541312d02e 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -73,7 +73,6 @@ public class StreamExpressionTest extends SolrCloudTestCase { private static final String COLLECTIONORALIAS = "collection1"; private static final String FILESTREAM_COLLECTION = "filestream_collection"; - private static final int TIMEOUT = DEFAULT_TIMEOUT; private static final String id = "id"; private static boolean useAlias; @@ -256,7 +255,7 @@ public void testCloudSolrStream() throws Exception { assertOrder(tuples, 0, 2, 1, 3, 4); assertLong(tuples.get(0), "a_i", 0); - // Execersise the /stream hander + // Exercise the /stream handler // Add the shards http parameter for the myCollection StringBuilder buf = new StringBuilder(); @@ -1040,7 +1039,7 @@ public void testStatsStream() throws Exception { assertEquals(6.0D, perf, 0.0); assertEquals(10, count, 0.0); - // Execersise the /stream hander + // Exercise the /stream handler // Add the shards http parameter for the myCollection StringBuilder buf = new StringBuilder(); @@ -1112,8 +1111,7 @@ public void testFacet2DStream() throws Exception { .add(id, "9", "diseases_s", "diabetes", "symptoms_s", "thirsty", "cases_i", "20") .add(id, "10", "diseases_s", "diabetes", "symptoms_s", "thirsty", "cases_i", "20") .commit(cluster.getSolrClient(), COLLECTIONORALIAS); - StreamExpression expression; - TupleStream stream; + List tuples; ModifiableSolrParams paramsLoc = new ModifiableSolrParams(); @@ -3667,15 +3665,6 @@ public void testSearchBacktick() throws Exception { assertEquals("l b c d color`s e", tuple2.get("test_t")); } - private Map getIdToLabel(TupleStream stream, String outField) throws IOException { - Map idToLabel = new HashMap<>(); - List tuples = getTuples(stream); - for (Tuple tuple : tuples) { - idToLabel.put(tuple.getString("id"), tuple.getDouble(outField)); - } - return idToLabel; - } - @Test public void testBasicTextLogitStream() throws Exception { Assume.assumeTrue(!useAlias); @@ -4047,7 +4036,7 @@ public void testSignificantTermsStream() throws Exception { assertEquals(5600, tuples.get(1).getLong("background").longValue()); assertEquals(5000, tuples.get(1).getLong("foreground").longValue()); - // Execersise the /stream hander + // Exercise the /stream handler // Add the shards http parameter for the myCollection StringBuilder buf = new StringBuilder(); @@ -4108,7 +4097,7 @@ public void tooLargeForGetRequest() throws IOException, SolrServerException { StreamContext streamContext = new StreamContext(); streamContext.setSolrClientCache(cache); // use filter() to allow being parsed as 'terms in set' query instead of a (weighted/scored) - // BooleanQuery so we don't trip too many boolean clauses + // BooleanQuery, so we don't trip too many boolean clauses String longQuery = "\"filter(id:(" + IntStream.range(0, 4000).mapToObj(i -> "a").collect(Collectors.joining(" ", "", "")) @@ -4374,7 +4363,7 @@ private static Path findUserFilesDataDir() { /** * Creates a tree of files underneath a provided data-directory. * - *

The filetree created looks like: + *

The file tree created looks like: * * 

    * dataDir
diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpressionTest.java
similarity index 98%
rename from solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
rename to solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpressionTest.java
index 2c941f142d7..37429f7a939 100644
--- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java
+++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpressionTest.java
@@ -30,11 +30,11 @@
 import org.junit.Test;
 
 /** */
-public class StreamExpressionToExpessionTest extends SolrTestCase {
+public class StreamExpressionToExpressionTest extends SolrTestCase {
 
-  private StreamFactory factory;
+  private final StreamFactory factory;
 
-  public StreamExpressionToExpessionTest() {
+  public StreamExpressionToExpressionTest() {
     super();
 
     factory =
@@ -77,7 +77,6 @@ public void testCloudSolrStream() throws Exception {
                 "search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\", fq=\"a_s:one\", fq=\"a_s:two\")"),
             factory)) {
       expressionString = stream.toExpression(factory).toString();
-      System.out.println("ExpressionString: " + expressionString.toString());
       assertTrue(expressionString.contains("search(collection1,"));
       assertTrue(expressionString.contains("q=\"*:*\""));
       assertTrue(expressionString.contains("fl=\"id,a_s,a_i,a_f\""));
@@ -550,7 +549,7 @@ public void testCloudSolrStreamWithEscapedQuote() throws Exception {
 
     // The purpose of this test is to ensure that a parameter with a contained " character is
     // properly escaped when it is turned back into an expression. This is important when an
-    // expression is passedto a worker (parallel stream) or even for other reasons when an
+    // expression is passed to a worker (parallel stream) or even for other reasons when an
     // expression is string-ified.
 
     // Basic test
diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
index d8770711876..42252add70c 100644
--- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
+++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
@@ -89,13 +89,12 @@ public class StreamingTest extends SolrCloudTestCase {
 
   private static String zkHost;
 
-  private static int numShards;
   private static int numWorkers;
   private static boolean useAlias;
 
   @BeforeClass
   public static void configureCluster() throws Exception {
-    numShards = random().nextInt(2) + 1; // 1 - 3
+    int numShards = random().nextInt(2) + 1; // 1 - 3
     numWorkers = numShards > 2 ? random().nextInt(numShards - 1) + 1 : numShards;
     configureCluster(numShards)
         .addConfig(

From 4ee1a48423ffc2d60617cee8c615095e98049623 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= 
Date: Tue, 16 Jul 2024 21:23:20 +0200
Subject: [PATCH 041/172] Let solrbot (renovate) upgrade github actions (#2559)

---
 .github/renovate.json | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/.github/renovate.json b/.github/renovate.json
index d59c413c1be..52f0d53d45e 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -3,8 +3,8 @@
   "description": "Runs Renovate with solrbot, see dev-docs/dependency-upgrades.adoc for more",
   "enabled": true,
   "dependencyDashboard": false,
-  "enabledManagers": ["gradle"],
-  "includePaths": ["versions.*", "build.gradle"],
+  "enabledManagers": ["gradle", "github-actions"],
+  "includePaths": ["versions.*", "build.gradle", ".github/workflows/*"],
   "postUpgradeTasks": {
     "commands": ["./gradlew updateLicenses"],
     "fileFilters": ["solr/licenses/*.sha1"],
@@ -14,7 +14,11 @@
     {
       "description": "Fix for non-semantic versions for older artifacts",
       "matchDatasources": ["maven"],
-      "matchPackageNames": ["commons-collections:commons-collections", "commons-io:commons-io", "commons-lang:commons-lang"],
+      "matchPackageNames": [
+        "commons-collections:commons-collections",
+        "commons-io:commons-io",
+        "commons-lang:commons-lang"
+      ],
       "versioning": "regex:^(?\\d{1,4})\\.(?\\d+)(\\.(?\\d+))?$"
     },
     {
@@ -55,29 +59,33 @@
     },
     {
       "description": "Changelog for commons-io",
-      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-io.git"],
-      "customChangelogUrl": "https://commons.apache.org/proper/commons-io/changes-report.html"
+      "matchSourceUrls": [
+        "https://gitbox.apache.org/repos/asf?p=commons-io.git"
+      ],
+      "changelogUrl": "https://commons.apache.org/proper/commons-io/changes-report.html"
     },
     {
       "description": "Changelog for zookeeper",
       "matchSourceUrls": ["https://gitbox.apache.org/repos/asf/zookeeper.git"],
-      "customChangelogUrl": "https://zookeeper.apache.org/releases.html"
+      "changelogUrl": "https://zookeeper.apache.org/releases.html"
     },
     {
       "description": "Changelog for commons-compress",
-      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-compress.git"],
-      "customChangelogUrl": "https://commons.apache.org/proper/commons-compress/changes-report.html"
+      "matchSourceUrls": [
+        "https://gitbox.apache.org/repos/asf?p=commons-compress.git"
+      ],
+      "changelogUrl": "https://commons.apache.org/proper/commons-compress/changes-report.html"
     },
     {
       "description": "Changelog for commons-configuration",
-      "matchSourceUrls": ["https://gitbox.apache.org/repos/asf?p=commons-configuration.git"],
-      "customChangelogUrl": "https://commons.apache.org/proper/commons-configuration/changes-report.html"
+      "matchSourceUrls": [
+        "https://gitbox.apache.org/repos/asf?p=commons-configuration.git"
+      ],
+      "changelogUrl": "https://commons.apache.org/proper/commons-configuration/changes-report.html"
     }
   ],
-  "schedule": [
-    "* * * * 0"
-  ],
+  "schedule": ["* * * * 0"],
   "prConcurrentLimit": 50,
   "prHourlyLimit": 10,
-  "stabilityDays": 5
+  "minimumReleaseAge": "5 days"
 }

From 864748acbdf6a8f9b6abca5237533c7089c3c2d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= 
Date: Tue, 16 Jul 2024 22:10:36 +0200
Subject: [PATCH 042/172] Renovatebot: Disable schedule and upper concurrent
 PRs This is temporary since it has not run for 3 weeks

---
 .github/renovate.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/renovate.json b/.github/renovate.json
index 52f0d53d45e..5cf53a4c5fd 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -84,8 +84,8 @@
       "changelogUrl": "https://commons.apache.org/proper/commons-configuration/changes-report.html"
     }
   ],
-  "schedule": ["* * * * 0"],
-  "prConcurrentLimit": 50,
+  "schedule": ["* * * * *"],
+  "prConcurrentLimit": 100,
   "prHourlyLimit": 10,
   "minimumReleaseAge": "5 days"
 }

From d45c923e3612bc94b34b67eef802dbdebe487a62 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Tue, 16 Jul 2024 23:43:16 +0200
Subject: [PATCH 043/172] Update actions/checkout action to v4 (#2562)

---
 .github/workflows/bin-solr-test.yml    | 2 +-
 .github/workflows/gradle-precommit.yml | 2 +-
 .github/workflows/solrj-test.yml       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml
index ed40a23680d..5f6a9a911d0 100644
--- a/.github/workflows/bin-solr-test.yml
+++ b/.github/workflows/bin-solr-test.yml
@@ -21,7 +21,7 @@ jobs:
 
     steps:
     # Setup
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up JDK 11
       uses: actions/setup-java@v2
       with:
diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml
index 309205d58f0..43419d4a687 100644
--- a/.github/workflows/gradle-precommit.yml
+++ b/.github/workflows/gradle-precommit.yml
@@ -17,7 +17,7 @@ jobs:
 
     steps:
     # Setup
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up JDK 11
       uses: actions/setup-java@v2
diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml
index 16710282b82..efe9969940e 100644
--- a/.github/workflows/solrj-test.yml
+++ b/.github/workflows/solrj-test.yml
@@ -20,7 +20,7 @@ jobs:
 
     steps:
     # Setup
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up JDK 11
       uses: actions/setup-java@v2
       with:

From ae9c60219db07746c1af3f52fa7ae2412fa6a6a3 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Tue, 16 Jul 2024 23:45:13 +0200
Subject: [PATCH 044/172] Update dependency com.fasterxml.jackson:jackson-bom
 to v2.17.2 (#2560)

---
 .../jackson-annotations-2.17.1.jar.sha1       |  1 -
 .../jackson-annotations-2.17.2.jar.sha1       |  1 +
 solr/licenses/jackson-core-2.17.1.jar.sha1    |  1 -
 solr/licenses/jackson-core-2.17.2.jar.sha1    |  1 +
 .../licenses/jackson-databind-2.17.1.jar.sha1 |  1 -
 .../licenses/jackson-databind-2.17.2.jar.sha1 |  1 +
 .../jackson-dataformat-cbor-2.17.1.jar.sha1   |  1 -
 .../jackson-dataformat-cbor-2.17.2.jar.sha1   |  1 +
 .../jackson-dataformat-smile-2.17.1.jar.sha1  |  1 -
 .../jackson-dataformat-smile-2.17.2.jar.sha1  |  1 +
 .../jackson-dataformat-xml-2.17.1.jar.sha1    |  1 -
 .../jackson-dataformat-xml-2.17.2.jar.sha1    |  1 +
 .../jackson-datatype-jdk8-2.17.1.jar.sha1     |  1 -
 .../jackson-datatype-jdk8-2.17.2.jar.sha1     |  1 +
 .../jackson-datatype-jsr310-2.17.1.jar.sha1   |  1 -
 .../jackson-datatype-jsr310-2.17.2.jar.sha1   |  1 +
 ...akarta-xmlbind-annotations-2.17.1.jar.sha1 |  1 -
 ...akarta-xmlbind-annotations-2.17.2.jar.sha1 |  1 +
 .../jackson-module-kotlin-2.17.1.jar.sha1     |  1 -
 .../jackson-module-kotlin-2.17.2.jar.sha1     |  1 +
 ...son-module-parameter-names-2.17.1.jar.sha1 |  1 -
 ...son-module-parameter-names-2.17.2.jar.sha1 |  1 +
 solr/licenses/woodstox-core-6.6.2.jar.sha1    |  1 -
 solr/licenses/woodstox-core-6.7.0.jar.sha1    |  1 +
 versions.lock                                 | 26 +++++++++----------
 versions.props                                |  2 +-
 26 files changed, 26 insertions(+), 26 deletions(-)
 delete mode 100644 solr/licenses/jackson-annotations-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-annotations-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-core-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-core-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-databind-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-databind-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-datatype-jdk8-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-datatype-jdk8-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-module-kotlin-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-module-kotlin-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/jackson-module-parameter-names-2.17.1.jar.sha1
 create mode 100644 solr/licenses/jackson-module-parameter-names-2.17.2.jar.sha1
 delete mode 100644 solr/licenses/woodstox-core-6.6.2.jar.sha1
 create mode 100644 solr/licenses/woodstox-core-6.7.0.jar.sha1

diff --git a/solr/licenses/jackson-annotations-2.17.1.jar.sha1 b/solr/licenses/jackson-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 4b004f804aa..00000000000
--- a/solr/licenses/jackson-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fca7ef6192c9ad05d07bc50da991bf937a84af3a
diff --git a/solr/licenses/jackson-annotations-2.17.2.jar.sha1 b/solr/licenses/jackson-annotations-2.17.2.jar.sha1
new file mode 100644
index 00000000000..d09733f1130
--- /dev/null
+++ b/solr/licenses/jackson-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+147b7b9412ffff24339f8aba080b292448e08698
diff --git a/solr/licenses/jackson-core-2.17.1.jar.sha1 b/solr/licenses/jackson-core-2.17.1.jar.sha1
deleted file mode 100644
index c9af041df12..00000000000
--- a/solr/licenses/jackson-core-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5e52a11644cd59a28ef79f02bddc2cc3bab45edb
diff --git a/solr/licenses/jackson-core-2.17.2.jar.sha1 b/solr/licenses/jackson-core-2.17.2.jar.sha1
new file mode 100644
index 00000000000..b6dd852e896
--- /dev/null
+++ b/solr/licenses/jackson-core-2.17.2.jar.sha1
@@ -0,0 +1 @@
+969a35cb35c86512acbadcdbbbfb044c877db814
diff --git a/solr/licenses/jackson-databind-2.17.1.jar.sha1 b/solr/licenses/jackson-databind-2.17.1.jar.sha1
deleted file mode 100644
index c0f5146bc1b..00000000000
--- a/solr/licenses/jackson-databind-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0524dcbcccdde7d45a679dfc333e4763feb09079
diff --git a/solr/licenses/jackson-databind-2.17.2.jar.sha1 b/solr/licenses/jackson-databind-2.17.2.jar.sha1
new file mode 100644
index 00000000000..924123949d7
--- /dev/null
+++ b/solr/licenses/jackson-databind-2.17.2.jar.sha1
@@ -0,0 +1 @@
+e6deb029e5901e027c129341fac39e515066b68c
diff --git a/solr/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1 b/solr/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
deleted file mode 100644
index b804e5277ee..00000000000
--- a/solr/licenses/jackson-dataformat-cbor-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba5d8e6ecc62aa0e49c0ce935b8696352dbebc71
diff --git a/solr/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1 b/solr/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
new file mode 100644
index 00000000000..7a74616b265
--- /dev/null
+++ b/solr/licenses/jackson-dataformat-cbor-2.17.2.jar.sha1
@@ -0,0 +1 @@
+57fa7c1b5104bbc4599278d13933a937ee058e68
diff --git a/solr/licenses/jackson-dataformat-smile-2.17.1.jar.sha1 b/solr/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
deleted file mode 100644
index 6b2ecc27a5c..00000000000
--- a/solr/licenses/jackson-dataformat-smile-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89683ac4f0a0c2c4f69ea56b90480ed40266dac8
diff --git a/solr/licenses/jackson-dataformat-smile-2.17.2.jar.sha1 b/solr/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
new file mode 100644
index 00000000000..f3ce7163082
--- /dev/null
+++ b/solr/licenses/jackson-dataformat-smile-2.17.2.jar.sha1
@@ -0,0 +1 @@
+20e956b9b6f67138edd39fab7a506ded19638bcb
diff --git a/solr/licenses/jackson-dataformat-xml-2.17.1.jar.sha1 b/solr/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
deleted file mode 100644
index f7003c58265..00000000000
--- a/solr/licenses/jackson-dataformat-xml-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e6a168dba62aa63743b9e2b83f4e0f0dfdc143d3
diff --git a/solr/licenses/jackson-dataformat-xml-2.17.2.jar.sha1 b/solr/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
new file mode 100644
index 00000000000..2ceffa565c8
--- /dev/null
+++ b/solr/licenses/jackson-dataformat-xml-2.17.2.jar.sha1
@@ -0,0 +1 @@
+ad58f5bd089e743ac6e5999b2d1e3cf8515cea9a
diff --git a/solr/licenses/jackson-datatype-jdk8-2.17.1.jar.sha1 b/solr/licenses/jackson-datatype-jdk8-2.17.1.jar.sha1
deleted file mode 100644
index a1495db87a9..00000000000
--- a/solr/licenses/jackson-datatype-jdk8-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-76b495194c36058904c82e288d285a1bd13f0ffa
diff --git a/solr/licenses/jackson-datatype-jdk8-2.17.2.jar.sha1 b/solr/licenses/jackson-datatype-jdk8-2.17.2.jar.sha1
new file mode 100644
index 00000000000..cc425cd38cc
--- /dev/null
+++ b/solr/licenses/jackson-datatype-jdk8-2.17.2.jar.sha1
@@ -0,0 +1 @@
+efd3dd0e1d0db8bc72abbe71c15e697bb83b4b45
diff --git a/solr/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1 b/solr/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
deleted file mode 100644
index bacb1dc25a5..00000000000
--- a/solr/licenses/jackson-datatype-jsr310-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0969b0c3cb8c75d759e9a6c585c44c9b9f3a4f75
diff --git a/solr/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1 b/solr/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
new file mode 100644
index 00000000000..dbb6c44b420
--- /dev/null
+++ b/solr/licenses/jackson-datatype-jsr310-2.17.2.jar.sha1
@@ -0,0 +1 @@
+267b85e9ba2892a37be6d80aa9ca1438a0d8c210
diff --git a/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.1.jar.sha1 b/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.1.jar.sha1
deleted file mode 100644
index 9306cb597bc..00000000000
--- a/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1119f6cb1dcafe770baac9924f9648397f1d52aa
diff --git a/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.2.jar.sha1 b/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.2.jar.sha1
new file mode 100644
index 00000000000..6e1e58e7a1f
--- /dev/null
+++ b/solr/licenses/jackson-module-jakarta-xmlbind-annotations-2.17.2.jar.sha1
@@ -0,0 +1 @@
+6c9723133a918f5777bde827ce6a6f1e4db935b2
diff --git a/solr/licenses/jackson-module-kotlin-2.17.1.jar.sha1 b/solr/licenses/jackson-module-kotlin-2.17.1.jar.sha1
deleted file mode 100644
index 2614773ad23..00000000000
--- a/solr/licenses/jackson-module-kotlin-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fa6fedd3afd097d2a11eeee38fa6e31bf85bd2d0
diff --git a/solr/licenses/jackson-module-kotlin-2.17.2.jar.sha1 b/solr/licenses/jackson-module-kotlin-2.17.2.jar.sha1
new file mode 100644
index 00000000000..a7ea821fbf4
--- /dev/null
+++ b/solr/licenses/jackson-module-kotlin-2.17.2.jar.sha1
@@ -0,0 +1 @@
+67d4e4ec0538f59a28c92f7fa0b63bd2c8b0ef21
diff --git a/solr/licenses/jackson-module-parameter-names-2.17.1.jar.sha1 b/solr/licenses/jackson-module-parameter-names-2.17.1.jar.sha1
deleted file mode 100644
index 6d675ced40e..00000000000
--- a/solr/licenses/jackson-module-parameter-names-2.17.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-74a998f6fbcedbddedf0a27e8ce72078b2e516a6
diff --git a/solr/licenses/jackson-module-parameter-names-2.17.2.jar.sha1 b/solr/licenses/jackson-module-parameter-names-2.17.2.jar.sha1
new file mode 100644
index 00000000000..e4d6cbc9e5f
--- /dev/null
+++ b/solr/licenses/jackson-module-parameter-names-2.17.2.jar.sha1
@@ -0,0 +1 @@
+d27b9f95ccce98984c1ba58d61c5a9c072b1ad95
diff --git a/solr/licenses/woodstox-core-6.6.2.jar.sha1 b/solr/licenses/woodstox-core-6.6.2.jar.sha1
deleted file mode 100644
index 668383ffd8a..00000000000
--- a/solr/licenses/woodstox-core-6.6.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0bfe905f19ce0e60bff5b7adcbded2681a092ac3
diff --git a/solr/licenses/woodstox-core-6.7.0.jar.sha1 b/solr/licenses/woodstox-core-6.7.0.jar.sha1
new file mode 100644
index 00000000000..0e961215105
--- /dev/null
+++ b/solr/licenses/woodstox-core-6.7.0.jar.sha1
@@ -0,0 +1 @@
+ebbbdee43bb8c49f673f35b5697ae6d8515a4711
diff --git a/versions.lock b/versions.lock
index 53c8d2dd898..9251ee6bf0d 100644
--- a/versions.lock
+++ b/versions.lock
@@ -6,14 +6,14 @@ com.carrotsearch:hppc:0.9.1 (2 constraints: ad0fc9a6)
 com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1 (2 constraints: cf1501e2)
 com.cybozu.labs:langdetect:1.1-20120112 (1 constraints: 5c066d5e)
 com.epam:parso:2.0.14 (1 constraints: 8e0c750e)
-com.fasterxml.jackson:jackson-bom:2.17.1 (12 constraints: aefcc272)
-com.fasterxml.jackson.core:jackson-annotations:2.17.1 (10 constraints: f9c26e07)
-com.fasterxml.jackson.core:jackson-core:2.17.1 (13 constraints: a0062708)
-com.fasterxml.jackson.core:jackson-databind:2.17.1 (18 constraints: ed67acfe)
-com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.17.1 (2 constraints: 641c9ff1)
-com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.17.1 (1 constraints: bb0eb066)
-com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.17.1 (2 constraints: ab2412e1)
-com.fasterxml.woodstox:woodstox-core:6.6.2 (2 constraints: a223ea84)
+com.fasterxml.jackson:jackson-bom:2.17.2 (12 constraints: bafcbe83)
+com.fasterxml.jackson.core:jackson-annotations:2.17.2 (10 constraints: ffc2cf0f)
+com.fasterxml.jackson.core:jackson-core:2.17.2 (13 constraints: a906f217)
+com.fasterxml.jackson.core:jackson-databind:2.17.2 (18 constraints: f667f314)
+com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.17.2 (2 constraints: 651ca0f1)
+com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.17.2 (1 constraints: bc0eb166)
+com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.17.2 (2 constraints: ac2451e1)
+com.fasterxml.woodstox:woodstox-core:6.7.0 (2 constraints: a123c684)
 com.github.ben-manes.caffeine:caffeine:3.1.8 (1 constraints: 0e050536)
 com.github.jai-imageio:jai-imageio-core:1.4.0 (1 constraints: 5c0ced01)
 com.github.junrar:junrar:7.5.3 (1 constraints: 660c1102)
@@ -389,11 +389,11 @@ com.amazonaws:aws-java-sdk-core:1.12.501 (2 constraints: b01a32b3)
 com.amazonaws:aws-java-sdk-kms:1.12.501 (1 constraints: 060dbd37)
 com.amazonaws:aws-java-sdk-s3:1.12.501 (1 constraints: 10136f43)
 com.amazonaws:jmespath-java:1.12.501 (2 constraints: b01a32b3)
-com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.1 (2 constraints: ab195913)
-com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.17.1 (3 constraints: fd2e96b4)
-com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.1 (4 constraints: 6e48db35)
-com.fasterxml.jackson.module:jackson-module-kotlin:2.17.1 (2 constraints: aa1d6b60)
-com.fasterxml.jackson.module:jackson-module-parameter-names:2.17.1 (2 constraints: 0d247f82)
+com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.17.2 (2 constraints: ac195a13)
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.17.2 (3 constraints: fe2ed2b4)
+com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.2 (4 constraints: 6f485c36)
+com.fasterxml.jackson.module:jackson-module-kotlin:2.17.2 (2 constraints: ab1d9860)
+com.fasterxml.jackson.module:jackson-module-parameter-names:2.17.2 (2 constraints: 0e24bb82)
 com.google.cloud:google-cloud-nio:0.127.3 (1 constraints: 9a0e5e6c)
 com.nimbusds:content-type:2.2 (1 constraints: d80b68eb)
 com.nimbusds:lang-tag:1.7 (1 constraints: dc0b6aeb)
diff --git a/versions.props b/versions.props
index 218be60a0f0..7b1b109ef4e 100644
--- a/versions.props
+++ b/versions.props
@@ -5,7 +5,7 @@ com.adobe.testing:s3mock-junit4=2.17.0
 com.carrotsearch.randomizedtesting:*=2.8.1
 com.carrotsearch:hppc=0.9.1
 com.cybozu.labs:langdetect=1.1-20120112
-com.fasterxml.jackson:jackson-bom=2.17.1
+com.fasterxml.jackson:jackson-bom=2.17.2
 com.github.ben-manes.caffeine:caffeine=3.1.8
 com.github.spotbugs:*=4.8.0
 com.github.stephenc.jcip:jcip-annotations=1.0-1

From 8748a65a831b6336cf18563f9ad6ecb4e95d3198 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Tue, 16 Jul 2024 23:50:21 +0200
Subject: [PATCH 045/172] Update org.eclipse.jetty*:* to v10.0.22 (#2468)

---
 solr/licenses/http2-client-10.0.20.jar.sha1   |  1 -
 solr/licenses/http2-client-10.0.22.jar.sha1   |  1 +
 solr/licenses/http2-common-10.0.20.jar.sha1   |  1 -
 solr/licenses/http2-common-10.0.22.jar.sha1   |  1 +
 solr/licenses/http2-hpack-10.0.20.jar.sha1    |  1 -
 solr/licenses/http2-hpack-10.0.22.jar.sha1    |  1 +
 ...tp2-http-client-transport-10.0.20.jar.sha1 |  1 -
 ...tp2-http-client-transport-10.0.22.jar.sha1 |  1 +
 solr/licenses/http2-server-10.0.20.jar.sha1   |  1 -
 solr/licenses/http2-server-10.0.22.jar.sha1   |  1 +
 .../jetty-alpn-client-10.0.20.jar.sha1        |  1 -
 .../jetty-alpn-client-10.0.22.jar.sha1        |  1 +
 .../jetty-alpn-java-client-10.0.20.jar.sha1   |  1 -
 .../jetty-alpn-java-client-10.0.22.jar.sha1   |  1 +
 .../jetty-alpn-java-server-10.0.20.jar.sha1   |  1 -
 .../jetty-alpn-java-server-10.0.22.jar.sha1   |  1 +
 .../jetty-alpn-server-10.0.20.jar.sha1        |  1 -
 .../jetty-alpn-server-10.0.22.jar.sha1        |  1 +
 solr/licenses/jetty-client-10.0.20.jar.sha1   |  1 -
 solr/licenses/jetty-client-10.0.22.jar.sha1   |  1 +
 solr/licenses/jetty-deploy-10.0.20.jar.sha1   |  1 -
 solr/licenses/jetty-deploy-10.0.22.jar.sha1   |  1 +
 solr/licenses/jetty-http-10.0.20.jar.sha1     |  1 -
 solr/licenses/jetty-http-10.0.22.jar.sha1     |  1 +
 solr/licenses/jetty-io-10.0.20.jar.sha1       |  1 -
 solr/licenses/jetty-io-10.0.22.jar.sha1       |  1 +
 solr/licenses/jetty-jmx-10.0.20.jar.sha1      |  1 -
 solr/licenses/jetty-jmx-10.0.22.jar.sha1      |  1 +
 solr/licenses/jetty-rewrite-10.0.20.jar.sha1  |  1 -
 solr/licenses/jetty-rewrite-10.0.22.jar.sha1  |  1 +
 solr/licenses/jetty-security-10.0.20.jar.sha1 |  1 -
 solr/licenses/jetty-security-10.0.22.jar.sha1 |  1 +
 solr/licenses/jetty-server-10.0.20.jar.sha1   |  1 -
 solr/licenses/jetty-server-10.0.22.jar.sha1   |  1 +
 solr/licenses/jetty-servlet-10.0.20.jar.sha1  |  1 -
 solr/licenses/jetty-servlet-10.0.22.jar.sha1  |  1 +
 solr/licenses/jetty-servlets-10.0.20.jar.sha1 |  1 -
 solr/licenses/jetty-servlets-10.0.22.jar.sha1 |  1 +
 .../jetty-start-10.0.20-shaded.jar.sha1       |  1 -
 .../jetty-start-10.0.22-shaded.jar.sha1       |  1 +
 solr/licenses/jetty-util-10.0.20.jar.sha1     |  1 -
 solr/licenses/jetty-util-10.0.22.jar.sha1     |  1 +
 solr/licenses/jetty-webapp-10.0.20.jar.sha1   |  1 -
 solr/licenses/jetty-webapp-10.0.22.jar.sha1   |  1 +
 solr/licenses/jetty-xml-10.0.20.jar.sha1      |  1 -
 solr/licenses/jetty-xml-10.0.22.jar.sha1      |  1 +
 versions.lock                                 | 44 +++++++++----------
 versions.props                                |  2 +-
 48 files changed, 46 insertions(+), 46 deletions(-)
 delete mode 100644 solr/licenses/http2-client-10.0.20.jar.sha1
 create mode 100644 solr/licenses/http2-client-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/http2-common-10.0.20.jar.sha1
 create mode 100644 solr/licenses/http2-common-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/http2-hpack-10.0.20.jar.sha1
 create mode 100644 solr/licenses/http2-hpack-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/http2-http-client-transport-10.0.20.jar.sha1
 create mode 100644 solr/licenses/http2-http-client-transport-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/http2-server-10.0.20.jar.sha1
 create mode 100644 solr/licenses/http2-server-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-alpn-client-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-alpn-client-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-alpn-java-client-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-alpn-java-client-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-alpn-java-server-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-alpn-java-server-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-alpn-server-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-alpn-server-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-client-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-client-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-deploy-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-deploy-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-http-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-http-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-io-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-io-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-jmx-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-jmx-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-rewrite-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-rewrite-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-security-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-security-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-server-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-server-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-servlet-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-servlet-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-servlets-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-servlets-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-start-10.0.20-shaded.jar.sha1
 create mode 100644 solr/licenses/jetty-start-10.0.22-shaded.jar.sha1
 delete mode 100644 solr/licenses/jetty-util-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-util-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-webapp-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-webapp-10.0.22.jar.sha1
 delete mode 100644 solr/licenses/jetty-xml-10.0.20.jar.sha1
 create mode 100644 solr/licenses/jetty-xml-10.0.22.jar.sha1

diff --git a/solr/licenses/http2-client-10.0.20.jar.sha1 b/solr/licenses/http2-client-10.0.20.jar.sha1
deleted file mode 100644
index c6ccfd12e03..00000000000
--- a/solr/licenses/http2-client-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-26af63a6baf2de1577cea4184201105435739250
diff --git a/solr/licenses/http2-client-10.0.22.jar.sha1 b/solr/licenses/http2-client-10.0.22.jar.sha1
new file mode 100644
index 00000000000..0625bf893c0
--- /dev/null
+++ b/solr/licenses/http2-client-10.0.22.jar.sha1
@@ -0,0 +1 @@
+ab56d4454a9724e62b2588164c884a6940c0e62a
diff --git a/solr/licenses/http2-common-10.0.20.jar.sha1 b/solr/licenses/http2-common-10.0.20.jar.sha1
deleted file mode 100644
index 0e47616ef51..00000000000
--- a/solr/licenses/http2-common-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-716d2c1932503ed92a46428ac687a3b5e9b392f7
diff --git a/solr/licenses/http2-common-10.0.22.jar.sha1 b/solr/licenses/http2-common-10.0.22.jar.sha1
new file mode 100644
index 00000000000..ae830c8b491
--- /dev/null
+++ b/solr/licenses/http2-common-10.0.22.jar.sha1
@@ -0,0 +1 @@
+06437d043c3e6265cf57ff287fc4877c4e8e3507
diff --git a/solr/licenses/http2-hpack-10.0.20.jar.sha1 b/solr/licenses/http2-hpack-10.0.20.jar.sha1
deleted file mode 100644
index 40f3ee9f2ba..00000000000
--- a/solr/licenses/http2-hpack-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-37b48f2b4b0e6ae0108bd4b388d35920fbe8cf37
diff --git a/solr/licenses/http2-hpack-10.0.22.jar.sha1 b/solr/licenses/http2-hpack-10.0.22.jar.sha1
new file mode 100644
index 00000000000..224ebb16783
--- /dev/null
+++ b/solr/licenses/http2-hpack-10.0.22.jar.sha1
@@ -0,0 +1 @@
+c05f5b48ae78a829e623ba87cf2f067ce1225fcc
diff --git a/solr/licenses/http2-http-client-transport-10.0.20.jar.sha1 b/solr/licenses/http2-http-client-transport-10.0.20.jar.sha1
deleted file mode 100644
index f8eb38fa162..00000000000
--- a/solr/licenses/http2-http-client-transport-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3824ed9e1c7fb1490595f84a2adc80fd0b29d647
diff --git a/solr/licenses/http2-http-client-transport-10.0.22.jar.sha1 b/solr/licenses/http2-http-client-transport-10.0.22.jar.sha1
new file mode 100644
index 00000000000..34157c529cb
--- /dev/null
+++ b/solr/licenses/http2-http-client-transport-10.0.22.jar.sha1
@@ -0,0 +1 @@
+8b5d6aab1bc2d9c19d070cd21885a6a2841698ec
diff --git a/solr/licenses/http2-server-10.0.20.jar.sha1 b/solr/licenses/http2-server-10.0.20.jar.sha1
deleted file mode 100644
index 444b6fb5652..00000000000
--- a/solr/licenses/http2-server-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2f37997e6ba2de73da2406d6b0c9ce2f2dbc0ec2
diff --git a/solr/licenses/http2-server-10.0.22.jar.sha1 b/solr/licenses/http2-server-10.0.22.jar.sha1
new file mode 100644
index 00000000000..0bd57199207
--- /dev/null
+++ b/solr/licenses/http2-server-10.0.22.jar.sha1
@@ -0,0 +1 @@
+4898419e9832dc6ccffc1fd44bfaaa239f2a8ed7
diff --git a/solr/licenses/jetty-alpn-client-10.0.20.jar.sha1 b/solr/licenses/jetty-alpn-client-10.0.20.jar.sha1
deleted file mode 100644
index 27b94993a47..00000000000
--- a/solr/licenses/jetty-alpn-client-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a45363717b22c3b2b8b2ecab4dc1840fd209374
diff --git a/solr/licenses/jetty-alpn-client-10.0.22.jar.sha1 b/solr/licenses/jetty-alpn-client-10.0.22.jar.sha1
new file mode 100644
index 00000000000..2c1cc2437fa
--- /dev/null
+++ b/solr/licenses/jetty-alpn-client-10.0.22.jar.sha1
@@ -0,0 +1 @@
+406aa96e86dc71ab62176ec1ed5212cd1898e38a
diff --git a/solr/licenses/jetty-alpn-java-client-10.0.20.jar.sha1 b/solr/licenses/jetty-alpn-java-client-10.0.20.jar.sha1
deleted file mode 100644
index 0c5f3ac1f9e..00000000000
--- a/solr/licenses/jetty-alpn-java-client-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e08cd3e4a05291be1f933ddff2207fe163e78523
diff --git a/solr/licenses/jetty-alpn-java-client-10.0.22.jar.sha1 b/solr/licenses/jetty-alpn-java-client-10.0.22.jar.sha1
new file mode 100644
index 00000000000..95552987baf
--- /dev/null
+++ b/solr/licenses/jetty-alpn-java-client-10.0.22.jar.sha1
@@ -0,0 +1 @@
+4b3abc80849dde116c20d84c834f0289bc61fc35
diff --git a/solr/licenses/jetty-alpn-java-server-10.0.20.jar.sha1 b/solr/licenses/jetty-alpn-java-server-10.0.20.jar.sha1
deleted file mode 100644
index 73e02e31895..00000000000
--- a/solr/licenses/jetty-alpn-java-server-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b638ff8ebe481b4448f218a51aec7f5ca6c031bc
diff --git a/solr/licenses/jetty-alpn-java-server-10.0.22.jar.sha1 b/solr/licenses/jetty-alpn-java-server-10.0.22.jar.sha1
new file mode 100644
index 00000000000..d60ca0fe045
--- /dev/null
+++ b/solr/licenses/jetty-alpn-java-server-10.0.22.jar.sha1
@@ -0,0 +1 @@
+2b5bbaae622721f78930009990c3d5a9118e6c3b
diff --git a/solr/licenses/jetty-alpn-server-10.0.20.jar.sha1 b/solr/licenses/jetty-alpn-server-10.0.20.jar.sha1
deleted file mode 100644
index 545589ec1c5..00000000000
--- a/solr/licenses/jetty-alpn-server-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c9b67f560a3f8563f9d27f59baa5afb3a3469ee4
diff --git a/solr/licenses/jetty-alpn-server-10.0.22.jar.sha1 b/solr/licenses/jetty-alpn-server-10.0.22.jar.sha1
new file mode 100644
index 00000000000..a2f2f494568
--- /dev/null
+++ b/solr/licenses/jetty-alpn-server-10.0.22.jar.sha1
@@ -0,0 +1 @@
+533246e6f8ab50186c52b1a9eee75f5259d05c06
diff --git a/solr/licenses/jetty-client-10.0.20.jar.sha1 b/solr/licenses/jetty-client-10.0.20.jar.sha1
deleted file mode 100644
index d6928cfe78c..00000000000
--- a/solr/licenses/jetty-client-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9580732622af180b37602c41e5967fd0b13322d1
diff --git a/solr/licenses/jetty-client-10.0.22.jar.sha1 b/solr/licenses/jetty-client-10.0.22.jar.sha1
new file mode 100644
index 00000000000..54c4ee02d3f
--- /dev/null
+++ b/solr/licenses/jetty-client-10.0.22.jar.sha1
@@ -0,0 +1 @@
+7946f242dab0d3e6bd699acb91a3c30bed6cf217
diff --git a/solr/licenses/jetty-deploy-10.0.20.jar.sha1 b/solr/licenses/jetty-deploy-10.0.20.jar.sha1
deleted file mode 100644
index af29bc497f6..00000000000
--- a/solr/licenses/jetty-deploy-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d02b181c5d99a2412e4bc78c6162e1cf58eaee68
diff --git a/solr/licenses/jetty-deploy-10.0.22.jar.sha1 b/solr/licenses/jetty-deploy-10.0.22.jar.sha1
new file mode 100644
index 00000000000..b9f52818703
--- /dev/null
+++ b/solr/licenses/jetty-deploy-10.0.22.jar.sha1
@@ -0,0 +1 @@
+e761608322a70bd80238572e7f16623e436f73bb
diff --git a/solr/licenses/jetty-http-10.0.20.jar.sha1 b/solr/licenses/jetty-http-10.0.20.jar.sha1
deleted file mode 100644
index b433519b4e6..00000000000
--- a/solr/licenses/jetty-http-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d76bf933db81e9f530f1e59f1858bceddb7c4a43
diff --git a/solr/licenses/jetty-http-10.0.22.jar.sha1 b/solr/licenses/jetty-http-10.0.22.jar.sha1
new file mode 100644
index 00000000000..892a6f5aa8c
--- /dev/null
+++ b/solr/licenses/jetty-http-10.0.22.jar.sha1
@@ -0,0 +1 @@
+5a82b1f99820e14bc31368e8a1d49ee018193e29
diff --git a/solr/licenses/jetty-io-10.0.20.jar.sha1 b/solr/licenses/jetty-io-10.0.20.jar.sha1
deleted file mode 100644
index 5f002595ba0..00000000000
--- a/solr/licenses/jetty-io-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-269cea7949f739e41707f3d574782c9f227fe188
diff --git a/solr/licenses/jetty-io-10.0.22.jar.sha1 b/solr/licenses/jetty-io-10.0.22.jar.sha1
new file mode 100644
index 00000000000..e4a790b9e8b
--- /dev/null
+++ b/solr/licenses/jetty-io-10.0.22.jar.sha1
@@ -0,0 +1 @@
+706353f75c6a97ae4bd90946bb594c7c56908213
diff --git a/solr/licenses/jetty-jmx-10.0.20.jar.sha1 b/solr/licenses/jetty-jmx-10.0.20.jar.sha1
deleted file mode 100644
index ddc778724d3..00000000000
--- a/solr/licenses/jetty-jmx-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a8468a1e42ceed57bb5280eca74154b99b803dc
diff --git a/solr/licenses/jetty-jmx-10.0.22.jar.sha1 b/solr/licenses/jetty-jmx-10.0.22.jar.sha1
new file mode 100644
index 00000000000..fa80fb655fa
--- /dev/null
+++ b/solr/licenses/jetty-jmx-10.0.22.jar.sha1
@@ -0,0 +1 @@
+726cb6525c0f46a13c2caac31004fe44df25fc86
diff --git a/solr/licenses/jetty-rewrite-10.0.20.jar.sha1 b/solr/licenses/jetty-rewrite-10.0.20.jar.sha1
deleted file mode 100644
index 733d241faec..00000000000
--- a/solr/licenses/jetty-rewrite-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0340f1d23ac712b6ca207b9dba6f2504a8722235
diff --git a/solr/licenses/jetty-rewrite-10.0.22.jar.sha1 b/solr/licenses/jetty-rewrite-10.0.22.jar.sha1
new file mode 100644
index 00000000000..18e392f49d6
--- /dev/null
+++ b/solr/licenses/jetty-rewrite-10.0.22.jar.sha1
@@ -0,0 +1 @@
+1910256ece821336b18ffbca47b2fc37042065e2
diff --git a/solr/licenses/jetty-security-10.0.20.jar.sha1 b/solr/licenses/jetty-security-10.0.20.jar.sha1
deleted file mode 100644
index 7afed86de5e..00000000000
--- a/solr/licenses/jetty-security-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f082f9198038e4781ff4cdf4d32db9f90fa8f38a
diff --git a/solr/licenses/jetty-security-10.0.22.jar.sha1 b/solr/licenses/jetty-security-10.0.22.jar.sha1
new file mode 100644
index 00000000000..d55aeacb25e
--- /dev/null
+++ b/solr/licenses/jetty-security-10.0.22.jar.sha1
@@ -0,0 +1 @@
+6987e8708990efdfe1878f6427fdd293fc0a8e9f
diff --git a/solr/licenses/jetty-server-10.0.20.jar.sha1 b/solr/licenses/jetty-server-10.0.20.jar.sha1
deleted file mode 100644
index a7940609a7e..00000000000
--- a/solr/licenses/jetty-server-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-66deea078236e21ae9dd7443716b7642766526aa
diff --git a/solr/licenses/jetty-server-10.0.22.jar.sha1 b/solr/licenses/jetty-server-10.0.22.jar.sha1
new file mode 100644
index 00000000000..7da9c89cba5
--- /dev/null
+++ b/solr/licenses/jetty-server-10.0.22.jar.sha1
@@ -0,0 +1 @@
+b4447ea127dff476960ab28c8f15aad2f70defee
diff --git a/solr/licenses/jetty-servlet-10.0.20.jar.sha1 b/solr/licenses/jetty-servlet-10.0.20.jar.sha1
deleted file mode 100644
index 84613614623..00000000000
--- a/solr/licenses/jetty-servlet-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e3661d28d5f2e2185fb085ce5560b2969e8f7c1e
diff --git a/solr/licenses/jetty-servlet-10.0.22.jar.sha1 b/solr/licenses/jetty-servlet-10.0.22.jar.sha1
new file mode 100644
index 00000000000..f5058977c0a
--- /dev/null
+++ b/solr/licenses/jetty-servlet-10.0.22.jar.sha1
@@ -0,0 +1 @@
+c794f52275e85432baf148b9c59bf3fa2c1f1043
diff --git a/solr/licenses/jetty-servlets-10.0.20.jar.sha1 b/solr/licenses/jetty-servlets-10.0.20.jar.sha1
deleted file mode 100644
index fdd4fe83066..00000000000
--- a/solr/licenses/jetty-servlets-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1262502a388a8b0f1c8574156b21310231030ab1
diff --git a/solr/licenses/jetty-servlets-10.0.22.jar.sha1 b/solr/licenses/jetty-servlets-10.0.22.jar.sha1
new file mode 100644
index 00000000000..89e7b1ee5b5
--- /dev/null
+++ b/solr/licenses/jetty-servlets-10.0.22.jar.sha1
@@ -0,0 +1 @@
+b90ba6757c3c56d3988b5c8d45087d181c1c5be0
diff --git a/solr/licenses/jetty-start-10.0.20-shaded.jar.sha1 b/solr/licenses/jetty-start-10.0.20-shaded.jar.sha1
deleted file mode 100644
index 37b822e95cb..00000000000
--- a/solr/licenses/jetty-start-10.0.20-shaded.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6dc548ee832030d365ef27f8d4342192e3eff6be
diff --git a/solr/licenses/jetty-start-10.0.22-shaded.jar.sha1 b/solr/licenses/jetty-start-10.0.22-shaded.jar.sha1
new file mode 100644
index 00000000000..6a2d564048d
--- /dev/null
+++ b/solr/licenses/jetty-start-10.0.22-shaded.jar.sha1
@@ -0,0 +1 @@
+6389146cd3e762d33a567fb721bf7060a07f03d8
diff --git a/solr/licenses/jetty-util-10.0.20.jar.sha1 b/solr/licenses/jetty-util-10.0.20.jar.sha1
deleted file mode 100644
index 57d0136a92d..00000000000
--- a/solr/licenses/jetty-util-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c628108266e75f17dffee34191b74a696fe21746
diff --git a/solr/licenses/jetty-util-10.0.22.jar.sha1 b/solr/licenses/jetty-util-10.0.22.jar.sha1
new file mode 100644
index 00000000000..5b5d826e6db
--- /dev/null
+++ b/solr/licenses/jetty-util-10.0.22.jar.sha1
@@ -0,0 +1 @@
+9add990f8bfe742ff807944113beb5990f14c1f0
diff --git a/solr/licenses/jetty-webapp-10.0.20.jar.sha1 b/solr/licenses/jetty-webapp-10.0.20.jar.sha1
deleted file mode 100644
index 7613cefc26e..00000000000
--- a/solr/licenses/jetty-webapp-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-62b3f061dda27e876f287b41ebce0502bd09e351
diff --git a/solr/licenses/jetty-webapp-10.0.22.jar.sha1 b/solr/licenses/jetty-webapp-10.0.22.jar.sha1
new file mode 100644
index 00000000000..6e035721282
--- /dev/null
+++ b/solr/licenses/jetty-webapp-10.0.22.jar.sha1
@@ -0,0 +1 @@
+a2517ec8dd2d0c623192094c379d20e5443b80b2
diff --git a/solr/licenses/jetty-xml-10.0.20.jar.sha1 b/solr/licenses/jetty-xml-10.0.20.jar.sha1
deleted file mode 100644
index 5a9352d5b62..00000000000
--- a/solr/licenses/jetty-xml-10.0.20.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aafebd811fdc4d12297814cdea8564341ddc295f
diff --git a/solr/licenses/jetty-xml-10.0.22.jar.sha1 b/solr/licenses/jetty-xml-10.0.22.jar.sha1
new file mode 100644
index 00000000000..813eb0bf0f9
--- /dev/null
+++ b/solr/licenses/jetty-xml-10.0.22.jar.sha1
@@ -0,0 +1 @@
+68d31a946fc73f8320766629f3a52cceb4d88ceb
diff --git a/versions.lock b/versions.lock
index 9251ee6bf0d..d51b1b9d0f8 100644
--- a/versions.lock
+++ b/versions.lock
@@ -279,28 +279,28 @@ org.codehaus.janino:janino:3.1.9 (1 constraints: 650dbd2c)
 org.codehaus.woodstox:stax2-api:4.2.2 (2 constraints: 38155daf)
 org.codelibs:jhighlight:1.1.0 (1 constraints: 590ce401)
 org.conscrypt:conscrypt-openjdk-uber:2.5.2 (1 constraints: ed0fea95)
-org.eclipse.jetty:jetty-alpn-client:10.0.20 (3 constraints: e52ef55d)
-org.eclipse.jetty:jetty-alpn-java-client:10.0.20 (2 constraints: 171b956d)
-org.eclipse.jetty:jetty-alpn-java-server:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty:jetty-alpn-server:10.0.20 (2 constraints: ed163d48)
-org.eclipse.jetty:jetty-client:10.0.20 (2 constraints: 171b956d)
-org.eclipse.jetty:jetty-deploy:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty:jetty-http:10.0.20 (6 constraints: 664fd287)
-org.eclipse.jetty:jetty-io:10.0.20 (9 constraints: 997d80d4)
-org.eclipse.jetty:jetty-jmx:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty:jetty-rewrite:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty:jetty-security:10.0.20 (2 constraints: b4136d78)
-org.eclipse.jetty:jetty-server:10.0.20 (7 constraints: ce6da919)
-org.eclipse.jetty:jetty-servlet:10.0.20 (2 constraints: 2e139d60)
-org.eclipse.jetty:jetty-servlets:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty:jetty-util:10.0.20 (9 constraints: 9981fa7f)
-org.eclipse.jetty:jetty-webapp:10.0.20 (2 constraints: 3c132561)
-org.eclipse.jetty:jetty-xml:10.0.20 (3 constraints: 0521c1b8)
-org.eclipse.jetty.http2:http2-client:10.0.20 (2 constraints: 171b956d)
-org.eclipse.jetty.http2:http2-common:10.0.20 (3 constraints: d324a55f)
-org.eclipse.jetty.http2:http2-hpack:10.0.20 (2 constraints: 1a15a6df)
-org.eclipse.jetty.http2:http2-http-client-transport:10.0.20 (1 constraints: 65058e40)
-org.eclipse.jetty.http2:http2-server:10.0.20 (1 constraints: 65058e40)
+org.eclipse.jetty:jetty-alpn-client:10.0.22 (3 constraints: eb2e095f)
+org.eclipse.jetty:jetty-alpn-java-client:10.0.22 (2 constraints: 1b1b156e)
+org.eclipse.jetty:jetty-alpn-java-server:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty:jetty-alpn-server:10.0.22 (2 constraints: f116a748)
+org.eclipse.jetty:jetty-client:10.0.22 (2 constraints: 1b1b156e)
+org.eclipse.jetty:jetty-deploy:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty:jetty-http:10.0.22 (6 constraints: 704fa48b)
+org.eclipse.jetty:jetty-io:10.0.22 (9 constraints: a97d68de)
+org.eclipse.jetty:jetty-jmx:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty:jetty-rewrite:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty:jetty-security:10.0.22 (2 constraints: b813c578)
+org.eclipse.jetty:jetty-server:10.0.22 (7 constraints: d86da720)
+org.eclipse.jetty:jetty-servlet:10.0.22 (2 constraints: 3213f360)
+org.eclipse.jetty:jetty-servlets:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty:jetty-util:10.0.22 (9 constraints: a7817c8b)
+org.eclipse.jetty:jetty-webapp:10.0.22 (2 constraints: 40137b61)
+org.eclipse.jetty:jetty-xml:10.0.22 (3 constraints: 0b21bdb9)
+org.eclipse.jetty.http2:http2-client:10.0.22 (2 constraints: 1b1b156e)
+org.eclipse.jetty.http2:http2-common:10.0.22 (3 constraints: d924c560)
+org.eclipse.jetty.http2:http2-hpack:10.0.22 (2 constraints: 1e1508e0)
+org.eclipse.jetty.http2:http2-http-client-transport:10.0.22 (1 constraints: 67059040)
+org.eclipse.jetty.http2:http2-server:10.0.22 (1 constraints: 67059040)
 org.eclipse.jetty.toolchain:jetty-servlet-api:4.0.6 (4 constraints: 883053bf)
 org.gagravarr:vorbis-java-core:0.8 (1 constraints: 010c57e9)
 org.gagravarr:vorbis-java-tika:0.8 (1 constraints: 010c57e9)
diff --git a/versions.props b/versions.props
index 7b1b109ef4e..e23b0847cb2 100644
--- a/versions.props
+++ b/versions.props
@@ -56,7 +56,7 @@ org.bitbucket.b_c:jose4j=0.9.6
 org.bouncycastle:bcpkix-jdk18on=1.77
 org.carrot2:carrot2-core=4.5.1
 org.codehaus.woodstox:stax2-api=4.2.2
-org.eclipse.jetty*:*=10.0.20
+org.eclipse.jetty*:*=10.0.22
 org.eclipse.jetty.toolchain:jetty-servlet-api=4.0.6
 org.glassfish.hk2*:*=3.0.5
 org.glassfish.hk2.external:jakarta.inject=2.6.1

From a848514c89b34a9a3af76c9868f211e5fae6cbb9 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Tue, 16 Jul 2024 23:53:04 +0200
Subject: [PATCH 046/172] Update org.apache.curator:* to v5.7.0 (#2182)

---
 solr/licenses/curator-client-5.5.0.jar.sha1    |  1 -
 solr/licenses/curator-client-5.7.0.jar.sha1    |  1 +
 solr/licenses/curator-framework-5.5.0.jar.sha1 |  1 -
 solr/licenses/curator-framework-5.7.0.jar.sha1 |  1 +
 solr/licenses/curator-recipes-5.5.0.jar.sha1   |  1 -
 solr/licenses/curator-recipes-5.7.0.jar.sha1   |  1 +
 versions.lock                                  | 10 +++++-----
 versions.props                                 |  2 +-
 8 files changed, 9 insertions(+), 9 deletions(-)
 delete mode 100644 solr/licenses/curator-client-5.5.0.jar.sha1
 create mode 100644 solr/licenses/curator-client-5.7.0.jar.sha1
 delete mode 100644 solr/licenses/curator-framework-5.5.0.jar.sha1
 create mode 100644 solr/licenses/curator-framework-5.7.0.jar.sha1
 delete mode 100644 solr/licenses/curator-recipes-5.5.0.jar.sha1
 create mode 100644 solr/licenses/curator-recipes-5.7.0.jar.sha1

diff --git a/solr/licenses/curator-client-5.5.0.jar.sha1 b/solr/licenses/curator-client-5.5.0.jar.sha1
deleted file mode 100644
index e9ae641df79..00000000000
--- a/solr/licenses/curator-client-5.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db2d83bdc0bac7b4f25fc113d8ce3eedc0a4e89c
diff --git a/solr/licenses/curator-client-5.7.0.jar.sha1 b/solr/licenses/curator-client-5.7.0.jar.sha1
new file mode 100644
index 00000000000..7bb1b27f325
--- /dev/null
+++ b/solr/licenses/curator-client-5.7.0.jar.sha1
@@ -0,0 +1 @@
+690e470ffe1c4bea8149d8eca9982842a447ffc6
diff --git a/solr/licenses/curator-framework-5.5.0.jar.sha1 b/solr/licenses/curator-framework-5.5.0.jar.sha1
deleted file mode 100644
index c5211cb22d0..00000000000
--- a/solr/licenses/curator-framework-5.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b706a216e49352103bd2527e83b1ec2410924494
diff --git a/solr/licenses/curator-framework-5.7.0.jar.sha1 b/solr/licenses/curator-framework-5.7.0.jar.sha1
new file mode 100644
index 00000000000..c826bcd8a3b
--- /dev/null
+++ b/solr/licenses/curator-framework-5.7.0.jar.sha1
@@ -0,0 +1 @@
+0b3a2c5191cb2aa609cc9bdd19701bdbd53ee627
diff --git a/solr/licenses/curator-recipes-5.5.0.jar.sha1 b/solr/licenses/curator-recipes-5.5.0.jar.sha1
deleted file mode 100644
index e19fcfe836c..00000000000
--- a/solr/licenses/curator-recipes-5.5.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4aa0cfb129c36cd91528fc1b8775705280e60285
diff --git a/solr/licenses/curator-recipes-5.7.0.jar.sha1 b/solr/licenses/curator-recipes-5.7.0.jar.sha1
new file mode 100644
index 00000000000..3b84a54317c
--- /dev/null
+++ b/solr/licenses/curator-recipes-5.7.0.jar.sha1
@@ -0,0 +1 @@
+6cd1b05889cc4ab8f074366dbd0493c6d3b87b6f
diff --git a/versions.lock b/versions.lock
index d51b1b9d0f8..a7180a56a67 100644
--- a/versions.lock
+++ b/versions.lock
@@ -44,7 +44,7 @@ com.google.cloud:google-cloud-storage:2.27.0 (2 constraints: d71c8a27)
 com.google.code.gson:gson:2.10.1 (7 constraints: 005f69b0)
 com.google.errorprone:error_prone_annotations:2.23.0 (11 constraints: be86b428)
 com.google.guava:failureaccess:1.0.1 (2 constraints: f9199e37)
-com.google.guava:guava:32.1.3-jre (26 constraints: 567bee8c)
+com.google.guava:guava:32.1.3-jre (26 constraints: b47b4d76)
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (2 constraints: 4b35b0a0)
 com.google.http-client:google-http-client:1.43.3 (11 constraints: 3fbf96b4)
 com.google.http-client:google-http-client-apache-v2:1.43.3 (2 constraints: b820e775)
@@ -180,9 +180,9 @@ org.apache.commons:commons-exec:1.4.0 (2 constraints: 031132cf)
 org.apache.commons:commons-lang3:3.14.0 (6 constraints: 404e54e4)
 org.apache.commons:commons-math3:3.6.1 (5 constraints: 57322799)
 org.apache.commons:commons-text:1.11.0 (1 constraints: da11b0f8)
-org.apache.curator:curator-client:5.5.0 (2 constraints: e81468a3)
-org.apache.curator:curator-framework:5.5.0 (2 constraints: 05144b75)
-org.apache.curator:curator-recipes:5.5.0 (1 constraints: 0c051336)
+org.apache.curator:curator-client:5.7.0 (2 constraints: ec14cea3)
+org.apache.curator:curator-framework:5.7.0 (2 constraints: 0914ad75)
+org.apache.curator:curator-recipes:5.7.0 (1 constraints: 0e051936)
 org.apache.hadoop:hadoop-annotations:3.3.6 (1 constraints: 0e050936)
 org.apache.hadoop:hadoop-auth:3.3.6 (1 constraints: 0e050936)
 org.apache.hadoop:hadoop-client-api:3.3.6 (3 constraints: 25280861)
@@ -259,7 +259,7 @@ org.apache.tika:tika-core:1.28.5 (2 constraints: d8118f11)
 org.apache.tika:tika-parsers:1.28.5 (1 constraints: 42054a3b)
 org.apache.tomcat:annotations-api:6.0.53 (1 constraints: 40054e3b)
 org.apache.xmlbeans:xmlbeans:5.0.3 (2 constraints: 72173075)
-org.apache.zookeeper:zookeeper:3.9.2 (2 constraints: 9e13a45f)
+org.apache.zookeeper:zookeeper:3.9.2 (2 constraints: a013aa5f)
 org.apache.zookeeper:zookeeper-jute:3.9.2 (2 constraints: 9d12b123)
 org.apiguardian:apiguardian-api:1.1.2 (2 constraints: 601bd5a8)
 org.bitbucket.b_c:jose4j:0.9.6 (1 constraints: 11050c36)
diff --git a/versions.props b/versions.props
index e23b0847cb2..8ad49619897 100644
--- a/versions.props
+++ b/versions.props
@@ -40,7 +40,7 @@ org.apache.commons:commons-configuration2=2.10.1
 org.apache.commons:commons-exec=1.4.0
 org.apache.commons:commons-lang3=3.14.0
 org.apache.commons:commons-math3=3.6.1
-org.apache.curator:*=5.5.0
+org.apache.curator:*=5.7.0
 org.apache.hadoop.thirdparty:*=1.1.1
 org.apache.hadoop:*=3.3.6
 org.apache.httpcomponents:httpclient=4.5.14

From da603bf477092725f9746a482c616da17a68e661 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Tue, 16 Jul 2024 23:56:36 +0200
Subject: [PATCH 047/172] Update org.slf4j:* to v2.0.13 (#2406)

---
 solr/licenses/jcl-over-slf4j-2.0.12.jar.sha1 | 1 -
 solr/licenses/jcl-over-slf4j-2.0.13.jar.sha1 | 1 +
 solr/licenses/jul-to-slf4j-2.0.12.jar.sha1   | 1 -
 solr/licenses/jul-to-slf4j-2.0.13.jar.sha1   | 1 +
 solr/licenses/slf4j-api-2.0.12.jar.sha1      | 1 -
 solr/licenses/slf4j-api-2.0.13.jar.sha1      | 1 +
 versions.lock                                | 6 +++---
 versions.props                               | 2 +-
 8 files changed, 7 insertions(+), 7 deletions(-)
 delete mode 100644 solr/licenses/jcl-over-slf4j-2.0.12.jar.sha1
 create mode 100644 solr/licenses/jcl-over-slf4j-2.0.13.jar.sha1
 delete mode 100644 solr/licenses/jul-to-slf4j-2.0.12.jar.sha1
 create mode 100644 solr/licenses/jul-to-slf4j-2.0.13.jar.sha1
 delete mode 100644 solr/licenses/slf4j-api-2.0.12.jar.sha1
 create mode 100644 solr/licenses/slf4j-api-2.0.13.jar.sha1

diff --git a/solr/licenses/jcl-over-slf4j-2.0.12.jar.sha1 b/solr/licenses/jcl-over-slf4j-2.0.12.jar.sha1
deleted file mode 100644
index d8b45c1f676..00000000000
--- a/solr/licenses/jcl-over-slf4j-2.0.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4462ddec70c0126e580dd0a637cb105a3c7a99ab
diff --git a/solr/licenses/jcl-over-slf4j-2.0.13.jar.sha1 b/solr/licenses/jcl-over-slf4j-2.0.13.jar.sha1
new file mode 100644
index 00000000000..69561f8af5b
--- /dev/null
+++ b/solr/licenses/jcl-over-slf4j-2.0.13.jar.sha1
@@ -0,0 +1 @@
+d062d6e35605aabee0c727a950e29d39ac0a262e
diff --git a/solr/licenses/jul-to-slf4j-2.0.12.jar.sha1 b/solr/licenses/jul-to-slf4j-2.0.12.jar.sha1
deleted file mode 100644
index 736a425d022..00000000000
--- a/solr/licenses/jul-to-slf4j-2.0.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb5f48f782b41cc881b0bf1fb4d88ae2ff6d5b93
diff --git a/solr/licenses/jul-to-slf4j-2.0.13.jar.sha1 b/solr/licenses/jul-to-slf4j-2.0.13.jar.sha1
new file mode 100644
index 00000000000..b17efe111d5
--- /dev/null
+++ b/solr/licenses/jul-to-slf4j-2.0.13.jar.sha1
@@ -0,0 +1 @@
+a3bcd9d9dd50c71ce69f06b1fd05e40fdeff6ba5
diff --git a/solr/licenses/slf4j-api-2.0.12.jar.sha1 b/solr/licenses/slf4j-api-2.0.12.jar.sha1
deleted file mode 100644
index 6319b346170..00000000000
--- a/solr/licenses/slf4j-api-2.0.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-48f109a2a6d8f446c794f3e3fa0d86df0cdfa312
diff --git a/solr/licenses/slf4j-api-2.0.13.jar.sha1 b/solr/licenses/slf4j-api-2.0.13.jar.sha1
new file mode 100644
index 00000000000..b4bee40b4da
--- /dev/null
+++ b/solr/licenses/slf4j-api-2.0.13.jar.sha1
@@ -0,0 +1 @@
+80229737f704b121a318bba5d5deacbcf395bc77
diff --git a/versions.lock b/versions.lock
index a7180a56a67..2450fe639ed 100644
--- a/versions.lock
+++ b/versions.lock
@@ -346,9 +346,9 @@ org.ow2.asm:asm-tree:7.2 (2 constraints: 2f14468c)
 org.quicktheories:quicktheories:0.26 (1 constraints: dc04f530)
 org.reactivestreams:reactive-streams:1.0.4 (3 constraints: 3f2b77fd)
 org.semver4j:semver4j:5.3.0 (1 constraints: 0a050d36)
-org.slf4j:jcl-over-slf4j:2.0.12 (3 constraints: f917acb5)
-org.slf4j:jul-to-slf4j:2.0.12 (3 constraints: 5328fa5e)
-org.slf4j:slf4j-api:2.0.12 (59 constraints: d5110b25)
+org.slf4j:jcl-over-slf4j:2.0.13 (3 constraints: fa17e8b5)
+org.slf4j:jul-to-slf4j:2.0.13 (3 constraints: 54285f5f)
+org.slf4j:slf4j-api:2.0.13 (59 constraints: d811902f)
 org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528)
 org.tallison:jmatio:1.5 (1 constraints: ff0b57e9)
 org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28)
diff --git a/versions.props b/versions.props
index 8ad49619897..83dba61bb96 100644
--- a/versions.props
+++ b/versions.props
@@ -71,6 +71,6 @@ org.openjdk.jmh:*=1.37
 org.osgi:osgi.annotation=8.1.0
 org.quicktheories:quicktheories=0.26
 org.semver4j:semver4j=5.3.0
-org.slf4j:*=2.0.12
+org.slf4j:*=2.0.13
 org.xerial.snappy:snappy-java=1.1.10.5
 software.amazon.awssdk:*=2.20.155

From 0a882e1ab59d74aaaff4dfdf5c6191620a449432 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Wed, 17 Jul 2024 00:06:21 +0200
Subject: [PATCH 048/172] Update io.netty:* to v4.1.111.Final (#2413)

---
 .../netty-buffer-4.1.108.Final.jar.sha1       |  1 -
 .../netty-buffer-4.1.111.Final.jar.sha1       |  1 +
 .../netty-codec-4.1.108.Final.jar.sha1        |  1 -
 .../netty-codec-4.1.111.Final.jar.sha1        |  1 +
 .../netty-codec-http-4.1.108.Final.jar.sha1   |  1 -
 .../netty-codec-http-4.1.111.Final.jar.sha1   |  1 +
 .../netty-codec-http2-4.1.108.Final.jar.sha1  |  1 -
 .../netty-codec-http2-4.1.111.Final.jar.sha1  |  1 +
 .../netty-codec-socks-4.1.108.Final.jar.sha1  |  1 -
 .../netty-codec-socks-4.1.111.Final.jar.sha1  |  1 +
 .../netty-common-4.1.108.Final.jar.sha1       |  1 -
 .../netty-common-4.1.111.Final.jar.sha1       |  1 +
 .../netty-handler-4.1.108.Final.jar.sha1      |  1 -
 .../netty-handler-4.1.111.Final.jar.sha1      |  1 +
 ...netty-handler-proxy-4.1.108.Final.jar.sha1 |  1 -
 ...netty-handler-proxy-4.1.111.Final.jar.sha1 |  1 +
 .../netty-resolver-4.1.108.Final.jar.sha1     |  1 -
 .../netty-resolver-4.1.111.Final.jar.sha1     |  1 +
 .../netty-transport-4.1.108.Final.jar.sha1    |  1 -
 .../netty-transport-4.1.111.Final.jar.sha1    |  1 +
 ...sport-classes-epoll-4.1.108.Final.jar.sha1 |  1 -
 ...sport-classes-epoll-4.1.111.Final.jar.sha1 |  1 +
 ...-epoll-4.1.108.Final-linux-x86_64.jar.sha1 |  1 -
 ...-epoll-4.1.111.Final-linux-x86_64.jar.sha1 |  1 +
 ...-native-unix-common-4.1.108.Final.jar.sha1 |  1 -
 ...-native-unix-common-4.1.111.Final.jar.sha1 |  1 +
 versions.lock                                 | 26 +++++++++----------
 versions.props                                |  2 +-
 28 files changed, 27 insertions(+), 27 deletions(-)
 delete mode 100644 solr/licenses/netty-buffer-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-buffer-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-codec-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-codec-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-codec-http-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-codec-http2-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-codec-socks-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-common-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-common-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-handler-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-handler-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-handler-proxy-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-resolver-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-resolver-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-transport-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-transport-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1
 delete mode 100644 solr/licenses/netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar.sha1
 create mode 100644 solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1
 delete mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.108.Final.jar.sha1
 create mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1

diff --git a/solr/licenses/netty-buffer-4.1.108.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.108.Final.jar.sha1
deleted file mode 100644
index f0ed1d75ab6..00000000000
--- a/solr/licenses/netty-buffer-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a9d06026ed251705e6ab52fa6ebe5f4f15aab7a
diff --git a/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..7502c1d0d11
--- /dev/null
+++ b/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+b54863f578939e135d3b3aea610284ae57c188cf
diff --git a/solr/licenses/netty-codec-4.1.108.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.108.Final.jar.sha1
deleted file mode 100644
index fd8db6d42c3..00000000000
--- a/solr/licenses/netty-codec-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c2ef6018eecde345fcddb96e31f651df16dca4c2
diff --git a/solr/licenses/netty-codec-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..96eec7829cc
--- /dev/null
+++ b/solr/licenses/netty-codec-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+a6762ec00a6d268f9980741f5b755838bcd658bf
diff --git a/solr/licenses/netty-codec-http-4.1.108.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.108.Final.jar.sha1
deleted file mode 100644
index 7e4f07f0b51..00000000000
--- a/solr/licenses/netty-codec-http-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dd44733e94f3f6237c896f2bbe9927c1eba48543
diff --git a/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..27b935424aa
--- /dev/null
+++ b/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+c6ecbc452321e632bf3cea0f9758839b650455c7
diff --git a/solr/licenses/netty-codec-http2-4.1.108.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.108.Final.jar.sha1
deleted file mode 100644
index 7b4b9e5a053..00000000000
--- a/solr/licenses/netty-codec-http2-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ed90430e545529a2df7c1db6c94568ea00867a61
diff --git a/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..b3c6f6aef26
--- /dev/null
+++ b/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+f0cca5df75bfb4f858d0435f601d8b1cae1de054
diff --git a/solr/licenses/netty-codec-socks-4.1.108.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.108.Final.jar.sha1
deleted file mode 100644
index c401e42cd40..00000000000
--- a/solr/licenses/netty-codec-socks-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ad0af28e408092f0d12994802a9f3fe18d45f8c
diff --git a/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..e932d399674
--- /dev/null
+++ b/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+ea52ef6617a9b69b0baaebb7f0b80373527f9607
diff --git a/solr/licenses/netty-common-4.1.108.Final.jar.sha1 b/solr/licenses/netty-common-4.1.108.Final.jar.sha1
deleted file mode 100644
index 0b0fbefd8e7..00000000000
--- a/solr/licenses/netty-common-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-30617b39cc6f850ca3807459fe726fbcd63989f2
diff --git a/solr/licenses/netty-common-4.1.111.Final.jar.sha1 b/solr/licenses/netty-common-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..c91f89a2dbf
--- /dev/null
+++ b/solr/licenses/netty-common-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+58210befcb31adbcadd5724966a061444db91863
diff --git a/solr/licenses/netty-handler-4.1.108.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.108.Final.jar.sha1
deleted file mode 100644
index 5f4da049d4b..00000000000
--- a/solr/licenses/netty-handler-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d186a0be320e6a139c42d9b018596ef9d4a0b4ca
diff --git a/solr/licenses/netty-handler-4.1.111.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..6f52f18a9bf
--- /dev/null
+++ b/solr/licenses/netty-handler-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+2bc6a58ad2e9e279634b6e55022e8dcd3c175cc4
diff --git a/solr/licenses/netty-handler-proxy-4.1.108.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.108.Final.jar.sha1
deleted file mode 100644
index c380943cc8c..00000000000
--- a/solr/licenses/netty-handler-proxy-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-62b6a5dfee2e22ab9015a469cb68e4727596fd4c
diff --git a/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..ecf253ebef6
--- /dev/null
+++ b/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+1e459c8630bb7c942b79a97e62dd728798de6a8c
diff --git a/solr/licenses/netty-resolver-4.1.108.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.108.Final.jar.sha1
deleted file mode 100644
index 464376ba30b..00000000000
--- a/solr/licenses/netty-resolver-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f3085568e45c2ca74118118f792d0d55968aeb13
diff --git a/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..a870fee5fa0
--- /dev/null
+++ b/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+3493179999f211dc49714319f81da2be86523a3b
diff --git a/solr/licenses/netty-transport-4.1.108.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.108.Final.jar.sha1
deleted file mode 100644
index 031c9530162..00000000000
--- a/solr/licenses/netty-transport-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1fd80f714c85ca685a80f32e0a4e8fd3b866e310
diff --git a/solr/licenses/netty-transport-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..61738b73629
--- /dev/null
+++ b/solr/licenses/netty-transport-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+24e97cf14ea9d80afe4c5ab69066b587fccc154a
diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.108.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.108.Final.jar.sha1
deleted file mode 100644
index dff7f5a21bf..00000000000
--- a/solr/licenses/netty-transport-classes-epoll-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-baf7b939ef71b25713cacbe47bef8caf80ce99c6
diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..8cca8c5497d
--- /dev/null
+++ b/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+8b97d32eb1489043e478deea99bd93ce487b82f6
diff --git a/solr/licenses/netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar.sha1
deleted file mode 100644
index ec009666bda..00000000000
--- a/solr/licenses/netty-transport-native-epoll-4.1.108.Final-linux-x86_64.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c62b2337352073ff41fa9a9857a53999d41b49b
diff --git a/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1
new file mode 100644
index 00000000000..2d45dd3c89a
--- /dev/null
+++ b/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1
@@ -0,0 +1 @@
+e102a0568f4f2cd7d4049c4154f94528cadf646b
diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.108.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.108.Final.jar.sha1
deleted file mode 100644
index e60f0dc7cfe..00000000000
--- a/solr/licenses/netty-transport-native-unix-common-4.1.108.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0df31f1cd96df8b2882b1e0faf4409b0bd704541
diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1
new file mode 100644
index 00000000000..689846d2410
--- /dev/null
+++ b/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1
@@ -0,0 +1 @@
+acafc128cddafa021bc0b48b0788eb0e118add5e
diff --git a/versions.lock b/versions.lock
index 2450fe639ed..1b7558e9260 100644
--- a/versions.lock
+++ b/versions.lock
@@ -110,21 +110,21 @@ io.grpc:grpc-services:1.61.1 (1 constraints: 211012a6)
 io.grpc:grpc-stub:1.61.1 (2 constraints: 5b1508d7)
 io.grpc:grpc-util:1.61.1 (4 constraints: 0222e69b)
 io.grpc:grpc-xds:1.61.1 (1 constraints: 211012a6)
-io.netty:netty-buffer:4.1.108.Final (10 constraints: 649b586c)
-io.netty:netty-codec:4.1.108.Final (5 constraints: 3c466293)
-io.netty:netty-codec-http:4.1.108.Final (3 constraints: d624f841)
-io.netty:netty-codec-http2:4.1.108.Final (1 constraints: 0f0b42d5)
-io.netty:netty-codec-socks:4.1.108.Final (1 constraints: 400fc77a)
-io.netty:netty-common:4.1.108.Final (12 constraints: 34b5f5a7)
-io.netty:netty-handler:4.1.108.Final (3 constraints: ef2b6ba5)
-io.netty:netty-handler-proxy:4.1.108.Final (1 constraints: 0f0b42d5)
-io.netty:netty-resolver:4.1.108.Final (2 constraints: b01a305e)
+io.netty:netty-buffer:4.1.111.Final (10 constraints: 289b3836)
+io.netty:netty-codec:4.1.111.Final (5 constraints: 1e469988)
+io.netty:netty-codec-http:4.1.111.Final (3 constraints: c4245f3e)
+io.netty:netty-codec-http2:4.1.111.Final (1 constraints: 0f0b42d5)
+io.netty:netty-codec-socks:4.1.111.Final (1 constraints: 3a0f9e7a)
+io.netty:netty-common:4.1.111.Final (12 constraints: ecb4455b)
+io.netty:netty-handler:4.1.111.Final (3 constraints: e32be3a1)
+io.netty:netty-handler-proxy:4.1.111.Final (1 constraints: 0f0b42d5)
+io.netty:netty-resolver:4.1.111.Final (2 constraints: a41ae85c)
 io.netty:netty-tcnative-boringssl-static:2.0.61.Final (1 constraints: d10fc38e)
 io.netty:netty-tcnative-classes:2.0.61.Final (1 constraints: d113ea5d)
-io.netty:netty-transport:4.1.108.Final (9 constraints: 858d97de)
-io.netty:netty-transport-classes-epoll:4.1.108.Final (1 constraints: dd12b130)
-io.netty:netty-transport-native-epoll:4.1.108.Final (1 constraints: 0310df9d)
-io.netty:netty-transport-native-unix-common:4.1.108.Final (4 constraints: fe3dc14b)
+io.netty:netty-transport:4.1.111.Final (9 constraints: 4f8d5ab2)
+io.netty:netty-transport-classes-epoll:4.1.111.Final (1 constraints: d7128830)
+io.netty:netty-transport-native-epoll:4.1.111.Final (1 constraints: 0310df9d)
+io.netty:netty-transport-native-unix-common:4.1.111.Final (4 constraints: ec3d7447)
 io.opencensus:opencensus-api:0.31.1 (5 constraints: 924d4692)
 io.opencensus:opencensus-contrib-http-util:0.31.1 (3 constraints: 7232a9fc)
 io.opencensus:opencensus-proto:0.2.0 (1 constraints: e60fd595)
diff --git a/versions.props b/versions.props
index 83dba61bb96..279429a2115 100644
--- a/versions.props
+++ b/versions.props
@@ -23,7 +23,7 @@ commons-collections:commons-collections=3.2.2
 commons-io:commons-io=2.15.1
 io.dropwizard.metrics:*=4.2.25
 io.grpc:grpc-*=1.61.1
-io.netty:*=4.1.108.Final
+io.netty:*=4.1.111.Final
 io.opentelemetry:opentelemetry-bom=1.35.0
 io.prometheus:*=0.16.0
 io.swagger.core.v3:*=2.2.22

From 647a74fc8e9f9416c87bd1f1c296274c0e4a3fb9 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Wed, 17 Jul 2024 17:59:16 +0200
Subject: [PATCH 049/172] Update actions/cache action to v4 (#2561)

---
 .github/workflows/bin-solr-test.yml    | 2 +-
 .github/workflows/gradle-precommit.yml | 2 +-
 .github/workflows/solrj-test.yml       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml
index 5f6a9a911d0..d1dd32eb0bf 100644
--- a/.github/workflows/bin-solr-test.yml
+++ b/.github/workflows/bin-solr-test.yml
@@ -30,7 +30,7 @@ jobs:
         java-package: jdk
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       with:
         path: |
           ~/.gradle/caches
diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml
index 43419d4a687..f8988238cca 100644
--- a/.github/workflows/gradle-precommit.yml
+++ b/.github/workflows/gradle-precommit.yml
@@ -29,7 +29,7 @@ jobs:
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       with:
         path: |
           ~/.gradle/caches
diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml
index efe9969940e..acb999a7270 100644
--- a/.github/workflows/solrj-test.yml
+++ b/.github/workflows/solrj-test.yml
@@ -29,7 +29,7 @@ jobs:
         java-package: jdk
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
-    - uses: actions/cache@v2
+    - uses: actions/cache@v4
       with:
         path: |
           ~/.gradle/caches

From 8d0ebbff15d8257429b749f7afeb8849b9fb8856 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Wed, 17 Jul 2024 17:59:49 +0200
Subject: [PATCH 050/172] Update actions/setup-java action to v4 (#2563)

---
 .github/workflows/bin-solr-test.yml    | 2 +-
 .github/workflows/gradle-precommit.yml | 2 +-
 .github/workflows/solrj-test.yml       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml
index d1dd32eb0bf..74c519b9929 100644
--- a/.github/workflows/bin-solr-test.yml
+++ b/.github/workflows/bin-solr-test.yml
@@ -23,7 +23,7 @@ jobs:
     # Setup
     - uses: actions/checkout@v4
     - name: Set up JDK 11
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11
diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml
index f8988238cca..06c1dd2a151 100644
--- a/.github/workflows/gradle-precommit.yml
+++ b/.github/workflows/gradle-precommit.yml
@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up JDK 11
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11
diff --git a/.github/workflows/solrj-test.yml b/.github/workflows/solrj-test.yml
index acb999a7270..4d615599db7 100644
--- a/.github/workflows/solrj-test.yml
+++ b/.github/workflows/solrj-test.yml
@@ -22,7 +22,7 @@ jobs:
     # Setup
     - uses: actions/checkout@v4
     - name: Set up JDK 11
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11

From 9a4c8a24898bf947aba1c1d153dd970c36db9899 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Wed, 17 Jul 2024 18:00:39 +0200
Subject: [PATCH 051/172] Update gradle/wrapper-validation-action action to v2
 (#2565)

---
 .github/workflows/gradle-precommit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml
index 06c1dd2a151..5c477134b2a 100644
--- a/.github/workflows/gradle-precommit.yml
+++ b/.github/workflows/gradle-precommit.yml
@@ -41,4 +41,4 @@ jobs:
     - name: Run gradle check (without tests)
       run: ./gradlew check -x test -Ptask.times=true
 
-    - uses: gradle/wrapper-validation-action@v1
+    - uses: gradle/wrapper-validation-action@v2

From d1905fcb053620edbd557b7e13558a42f6527835 Mon Sep 17 00:00:00 2001
From: Solr Bot <125606113+solrbot@users.noreply.github.com>
Date: Wed, 17 Jul 2024 18:01:28 +0200
Subject: [PATCH 052/172] Update actions/upload-artifact action to v4 (#2564)

---
 .github/workflows/bin-solr-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml
index 74c519b9929..d0cd3ddb52c 100644
--- a/.github/workflows/bin-solr-test.yml
+++ b/.github/workflows/bin-solr-test.yml
@@ -42,7 +42,7 @@ jobs:
       run: ./gradlew integrationTests
     - name: Archive logs
       if: ${{ failure() }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: logs
         path: solr/packaging/build/test-output

From f3f441c5300c53434fadfab1c26caa27c5516999 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= 
Date: Wed, 17 Jul 2024 20:23:38 +0200
Subject: [PATCH 053/172] SOLR-17357: Improve SolrCLI tool --help printout
 (#2545)

---
 .../org/apache/solr/cli/RunExampleTool.java   |  16 +--
 .../src/java/org/apache/solr/cli/SolrCLI.java |  39 ++++--
 .../src/java/org/apache/solr/cli/Tool.java    |  25 ++++
 .../org/apache/solr/cli/AuthToolTest.java     |   2 +-
 .../org/apache/solr/cli/CreateToolTest.java   |   2 +-
 .../org/apache/solr/cli/DeleteToolTest.java   |   2 +-
 .../apache/solr/cli/HealthcheckToolTest.java  |   2 +-
 .../org/apache/solr/cli/PackageToolTest.java  |   2 +-
 .../org/apache/solr/cli/PostToolTest.java     |   2 +-
 .../apache/solr/cli/SolrCLIZkToolsTest.java   | 112 +++++++-----------
 .../org/apache/solr/cli/TestExportTool.java   |   2 +-
 .../solr/cli/TestSolrCLIRunExample.java       |  16 +--
 .../apache/solr/cli/ZkSubcommandsTest.java    |   2 +-
 .../solr/cloud/SolrCloudExampleTest.java      |   9 +-
 .../security/BasicAuthIntegrationTest.java    |   2 +-
 .../solr/cloud/AbstractDistribZkTestBase.java |   2 +-
 16 files changed, 117 insertions(+), 120 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index d9fc740ddcd..8a6163f0945 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -320,10 +320,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
             "--solr-url", solrUrl
           };
       CreateTool createTool = new CreateTool(stdout);
-      int createCode =
-          createTool.runTool(
-              SolrCLI.processCommandLineArgs(
-                  createTool.getName(), createTool.getOptions(), createArgs));
+      int createCode = createTool.runTool(SolrCLI.processCommandLineArgs(createTool, createArgs));
       if (createCode != 0)
         throw new Exception(
             "Failed to create " + collectionName + " using command: " + Arrays.asList(createArgs));
@@ -353,8 +350,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
               exampledocsDir.getAbsolutePath() + "/*.xml"
             };
         PostTool postTool = new PostTool();
-        CommandLine postToolCli =
-            SolrCLI.parseCmdLine(postTool.getName(), args, postTool.getOptions());
+        CommandLine postToolCli = SolrCLI.parseCmdLine(postTool, args);
         postTool.runTool(postToolCli);
 
       } else {
@@ -435,8 +431,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
               filmsJsonFile.getAbsolutePath()
             };
         PostTool postTool = new PostTool();
-        CommandLine postToolCli =
-            SolrCLI.parseCmdLine(postTool.getName(), args, postTool.getOptions());
+        CommandLine postToolCli = SolrCLI.parseCmdLine(postTool, args);
         postTool.runTool(postToolCli);
 
       } catch (Exception ex) {
@@ -875,10 +870,7 @@ protected String createCloudExampleCollection(
         };
 
     CreateTool createTool = new CreateTool(stdout);
-    int createCode =
-        createTool.runTool(
-            SolrCLI.processCommandLineArgs(
-                createTool.getName(), createTool.getOptions(), createArgs));
+    int createCode = createTool.runTool(SolrCLI.processCommandLineArgs(createTool, createArgs));
 
     if (createCode != 0)
       throw new Exception(
diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
index 06e0714b722..8463c1529e1 100755
--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java
@@ -205,7 +205,7 @@ public static void main(String[] args) throws Exception {
       CLIO.err(iae.getMessage());
       System.exit(1);
     }
-    CommandLine cli = parseCmdLine(tool.getName(), args, tool.getOptions());
+    CommandLine cli = parseCmdLine(tool, args);
     System.exit(tool.runTool(cli));
   }
 
@@ -214,7 +214,7 @@ public static Tool findTool(String[] args) throws Exception {
     return newTool(toolType);
   }
 
-  public static CommandLine parseCmdLine(String toolName, String[] args, List