Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SAI-4837 : In-memory synthetic SolrCore for QA node #181

Merged
merged 20 commits into from
May 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,20 @@ public ZkConfigSetService(SolrZkClient zkClient) {
this.zkClient = zkClient;
}

/**
* Do not perform Zk operations if configSetName is provided.
*
* <p>This is only used by {@link org.apache.solr.core.SyntheticSolrCore}, which is not registered
* with Zookeeper
*/
@Override
protected SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd, String configSetName) {
return configSetName != null
? new ZkSolrResourceLoader(
cd.getInstanceDir(), configSetName, parentLoader.getClassLoader(), zkController)
: createCoreResourceLoader(cd);
}

@Override
public SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {
final String colName = cd.getCollectionName();
Expand Down
27 changes: 26 additions & 1 deletion solr/core/src/java/org/apache/solr/core/ConfigSetService.java
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,19 @@ public boolean isConfigSetTrusted(SolrResourceLoader coreLoader) throws IOExcept
* @return a ConfigSet
*/
public final ConfigSet loadConfigSet(CoreDescriptor dcore) {
return loadConfigSet(dcore, null);
}

/**
* Load the ConfigSet for a core with an explicit config set name
*
* @param dcore the core's CoreDescriptor
* @param configSetName an optional and explicit config set name
* @return a ConfigSet
*/
final ConfigSet loadConfigSet(CoreDescriptor dcore, String configSetName) {

SolrResourceLoader coreLoader = createCoreResourceLoader(dcore);
SolrResourceLoader coreLoader = createCoreResourceLoader(dcore, configSetName);

try {
// ConfigSet properties are loaded from ConfigSetProperties.DEFAULT_FILENAME file.
Expand Down Expand Up @@ -391,6 +402,20 @@ protected NamedList<Object> loadConfigSetFlags(SolrResourceLoader loader) throws
*/
protected abstract SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd);

/**
* Create a SolrResourceLoader for a core with the provided configSetName.
*
* <p>By default, this will just call {@link
* ConfigSetService#createConfigSetService(CoreContainer)}. Child implementation might override
* this to make use of the configSetName directly
*
* @param cd the core's CoreDescriptor
* @param configSetName an optional config set name
* @return a SolrResourceLoader
*/
protected SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd, String configSetName) {
return createCoreResourceLoader(cd);
}
/**
* Return a name for the ConfigSet for a core to be used for printing/diagnostic purposes.
*
Expand Down
2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/core/SolrCore.java
Original file line number Diff line number Diff line change
Expand Up @@ -1255,7 +1255,7 @@ public void seedVersionBuckets() {
}

/** Set UpdateLog to buffer updates if the slice is in construction. */
private void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) {
protected void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) {

if (coreContainer != null && coreContainer.isZooKeeperAware()) {
if (reqHandlers.get("/get") == null) {
Expand Down
66 changes: 66 additions & 0 deletions solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.apache.solr.core;

import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.rest.RestManager;

/**
* A synthetic core that is created only in memory and not registered against Zookeeper.
*
* <p>This is only used in Coordinator node to support a subset of SolrCore functionalities required
* by Coordinator operations such as aggregating and writing out response and providing configset
* info.
*
* <p>There should only be one instance of SyntheticSolrCore per configset
*/
public class SyntheticSolrCore extends SolrCore {
public SyntheticSolrCore(CoreContainer coreContainer, CoreDescriptor cd, ConfigSet configSet) {
super(coreContainer, cd, configSet);
}

public static SyntheticSolrCore createAndRegisterCore(
CoreContainer coreContainer, String syntheticCoreName, String configSetName) {
Map<String, String> coreProps = new HashMap<>();
coreProps.put(CoreAdminParams.CORE_NODE_NAME, coreContainer.getHostName());
coreProps.put(CoreAdminParams.COLLECTION, syntheticCoreName);

CoreDescriptor syntheticCoreDescriptor =
new CoreDescriptor(
syntheticCoreName,
Paths.get(coreContainer.getSolrHome() + "/" + syntheticCoreName),
coreProps,
coreContainer.getContainerProperties(),
coreContainer.getZkController());

ConfigSet coreConfig =
coreContainer.getConfigSetService().loadConfigSet(syntheticCoreDescriptor, configSetName);
syntheticCoreDescriptor.setConfigSetTrusted(coreConfig.isTrusted());
SyntheticSolrCore syntheticCore =
new SyntheticSolrCore(coreContainer, syntheticCoreDescriptor, coreConfig);
coreContainer.registerCore(syntheticCoreDescriptor, syntheticCore, false, false);

return syntheticCore;
}

@Override
protected void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) {
// no updates to SyntheticSolrCore
}

@Override
protected RestManager initRestManager() throws SolrException {
// returns an initialized RestManager. As init routines requires reading configname of the
// core's collection from ZK
// which synthetic core is not registered in ZK.
// We do not expect RestManager ops on Coordinator Nodes
return new RestManager();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what restManager enables?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, there is only a tiny feature called "managed resources" being powered by RestManager. It's not even reuired

}

@Override
public void close() {
super.close();
}
}
Loading
Loading