From 7a12a63997b5a7737062012a68d78adc0a912e75 Mon Sep 17 00:00:00 2001 From: patsonluk Date: Fri, 23 Feb 2024 20:32:57 -0800 Subject: [PATCH] Experimental QA core as proxy in memory --- .../apache/solr/cloud/ZkConfigSetService.java | 6 + .../java/org/apache/solr/core/SolrCore.java | 15 ++- .../solr/servlet/CoordinatorHttpSolrCall.java | 119 +----------------- 3 files changed, 24 insertions(+), 116 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java index 5f83f88c8eb..86a088ecef7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java @@ -66,6 +66,12 @@ public ZkConfigSetService(SolrZkClient zkClient) { this.zkClient = zkClient; } + @Override + protected SolrResourceLoader createConfigSetResourceLoader(CoreDescriptor cd, String configSetName) { + return new ZkSolrResourceLoader( + cd.getInstanceDir(), configSetName, parentLoader.getClassLoader(), zkController); + } + @Override public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) { final String colName = cd.getCollectionName(); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index f4d8309205b..b07f7f1ec83 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -154,6 +154,7 @@ import org.apache.solr.search.facet.FacetParserFactory; import org.apache.solr.search.stats.LocalStatsCache; import org.apache.solr.search.stats.StatsCache; +import org.apache.solr.servlet.CoordinatorHttpSolrCall; import org.apache.solr.update.DefaultSolrCoreState; import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.update.IndexFingerprint; @@ -1052,7 +1053,9 @@ public CoreContainer getCoreContainer() { return coreContainer; } - SolrCore(CoreContainer coreContainer, CoreDescriptor cd, ConfigSet configSet) { + + //TODO was protected. a proper way will likely be creating a SolrCoreProxy class + public SolrCore(CoreContainer coreContainer, CoreDescriptor cd, ConfigSet configSet) { this(coreContainer, cd, configSet, null, null, null, null, false); } @@ -1175,7 +1178,7 @@ private SolrCore( initSearcher(prev); // Initialize the RestManager - restManager = initRestManager(); + restManager = isSynthetic() ? new RestManager() : initRestManager(); // Finally tell anyone who wants to know resourceLoader.inform(resourceLoader); @@ -1202,7 +1205,9 @@ private SolrCore( // searcher! seedVersionBuckets(); - bufferUpdatesIfConstructing(coreDescriptor); + if (!isSynthetic()) { + bufferUpdatesIfConstructing(coreDescriptor); + } this.ruleExpiryLock = new ReentrantLock(); this.snapshotDelLock = new ReentrantLock(); @@ -3095,6 +3100,10 @@ public void fetchLatestSchema() { setLatestSchema(schema); } + public final boolean isSynthetic() { + return coreDescriptor.getCollectionName() != null && coreDescriptor.getCollectionName().startsWith(CoordinatorHttpSolrCall.SYNTHETIC_COLL_PREFIX); + } + public interface RawWriter { default String getContentType() { return BinaryResponseParser.BINARY_CONTENT_TYPE; diff --git a/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java index a2759b3edb8..486999d2ef8 100644 --- a/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/CoordinatorHttpSolrCall.java @@ -89,8 +89,8 @@ public static SolrCore getCore( ZkStateReader zkStateReader = solrCall.cores.getZkController().getZkStateReader(); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollectionOrNull(collectionName, true); - SolrCore core = null; - if (coll != null) { + + synchronized (CoordinatorHttpSolrCall.class) { String confName = coll.getConfigName(); String syntheticCollectionName = getSyntheticCollectionName(confName); @@ -113,118 +113,9 @@ public static SolrCore getCore( //after this point the sync core should be available in the container. Double check if (coreContainer.getCore(syntheticCore.getName()) != null) { - factory.collectionVsCoreNameMapping.put(collectionName, core.getName()); + factory.collectionVsCoreNameMapping.put(collectionName, syntheticCore.getName()); return syntheticCore; } - - - DocCollection syntheticColl = clusterState.getCollectionOrNull(syntheticCollectionName); - synchronized (CoordinatorHttpSolrCall.class) { - if (syntheticColl == null) { - // no synthetic collection for this config, let's create one - if (log.isInfoEnabled()) { - log.info( - "synthetic collection: {} does not exist, creating.. ", syntheticCollectionName); - } - - SolrException createException = null; - try { - createColl(syntheticCollectionName, solrCall.cores, confName); - } catch (SolrException exception) { - // concurrent requests could have created the collection hence causing collection - // exists - // exception - createException = exception; - } finally { - syntheticColl = - zkStateReader.getClusterState().getCollectionOrNull(syntheticCollectionName); - } - - // then indeed the collection was not created properly, either by this or other - // concurrent - // requests - if (syntheticColl == null) { - if (createException != null) { - throw createException; // rethrow the exception since such collection was not - // created - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Could not locate synthetic collection [" - + syntheticCollectionName - + "] after creation!"); - } - } - } - - // get docCollection again to ensure we get the fresh state - syntheticColl = - zkStateReader.getClusterState().getCollectionOrNull(syntheticCollectionName); - List nodeNameSyntheticReplicas = - syntheticColl.getReplicas(solrCall.cores.getZkController().getNodeName()); - if (nodeNameSyntheticReplicas == null || nodeNameSyntheticReplicas.isEmpty()) { - // this node does not have a replica. add one - if (log.isInfoEnabled()) { - log.info( - "this node does not have a replica of the synthetic collection: {} , adding replica ", - syntheticCollectionName); - } - - addReplica(syntheticCollectionName, solrCall.cores); - } - - // still have to ensure that it's active, otherwise super.getCoreByCollection - // will return null and then CoordinatorHttpSolrCall will call getCore again - // hence creating a calling loop - try { - zkStateReader.waitForState( - syntheticCollectionName, - 10, - TimeUnit.SECONDS, - docCollection -> { - for (Replica nodeNameSyntheticReplica : - docCollection.getReplicas(solrCall.cores.getZkController().getNodeName())) { - if (nodeNameSyntheticReplica.getState() == Replica.State.ACTIVE) { - return true; - } - } - return false; - }); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to wait for active replica for synthetic collection [" - + syntheticCollectionName - + "]", - e); - } - } - - core = solrCall.getCoreByCollection(syntheticCollectionName, isPreferLeader); - if (core != null) { - factory.collectionVsCoreNameMapping.put(collectionName, core.getName()); - // for the watcher, only remove on collection deletion (ie collection == null), since - // watch from coordinator is collection specific - solrCall - .cores - .getZkController() - .getZkStateReader() - .registerDocCollectionWatcher( - collectionName, - collection -> { - if (collection == null) { - factory.collectionVsCoreNameMapping.remove(collectionName); - return true; - } else { - return false; - } - }); - if (log.isDebugEnabled()) { - log.debug("coordinator node, returns synthetic core: {}", core.getName()); - } - } - setMDCLoggingContext(collectionName); - return core; } return null; } @@ -308,7 +199,9 @@ protected void init() throws Exception { public static SolrQueryRequest wrappedReq( SolrQueryRequest delegate, String collectionName, HttpSolrCall httpSolrCall) { Properties p = new Properties(); - p.put(CoreDescriptor.CORE_COLLECTION, collectionName); + if (collectionName != null) { + p.put(CoreDescriptor.CORE_COLLECTION, collectionName); + } p.put(CloudDescriptor.REPLICA_TYPE, Replica.Type.PULL.toString()); p.put(CoreDescriptor.CORE_SHARD, "_");