Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-16654: Add support for node-level caches (apache/solr#1351) #138

Open
wants to merge 2 commits into
base: fs/branch_9_3
Choose a base branch
from
Open

SOLR-16654: Add support for node-level caches (apache/solr#1351) #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c642469
SOLR-16654: Add support for node-level caches (#1351)
magibney Aug 30, 2023
c3efa80
conform to FS modified SolrCache ctor signature
magibney Sep 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions solr/core/src/java/org/apache/solr/core/CoreContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -96,6 +97,7 @@
import org.apache.solr.common.cloud.Replica.State;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectCache;
Expand Down Expand Up @@ -138,6 +140,8 @@
import org.apache.solr.pkg.SolrPackageLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.SolrFieldCacheBean;
import org.apache.solr.security.AllowListUrlChecker;
import org.apache.solr.security.AuditLoggerPlugin;
Expand Down Expand Up @@ -274,6 +278,8 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() {

private volatile SolrClientCache solrClientCache;

private volatile Map<String, SolrCache<?, ?>> caches;

private final ObjectCache objectCache = new ObjectCache();

public final NodeRoles nodeRoles = new NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP));
Expand Down Expand Up @@ -712,6 +718,10 @@ public PackageStoreAPI getPackageStoreAPI() {
return packageStoreAPI;
}

public SolrCache<?, ?> getCache(String name) {
return caches.get(name);
}

public SolrClientCache getSolrClientCache() {
return solrClientCache;
}
Expand Down Expand Up @@ -798,6 +808,20 @@ private void loadInternal() {

solrClientCache = new SolrClientCache(updateShardHandler.getDefaultHttpClient());

Map<String, CacheConfig> cachesConfig = cfg.getCachesConfig();
if (cachesConfig.isEmpty()) {
this.caches = Collections.emptyMap();
} else {
Map<String, SolrCache<?, ?>> m = CollectionUtil.newHashMap(cachesConfig.size());
for (Map.Entry<String, CacheConfig> e : cachesConfig.entrySet()) {
SolrCache<?, ?> c = e.getValue().newInstance(null);
String cacheName = e.getKey();
c.initializeMetrics(solrMetricsContext, "nodeLevelCache/" + cacheName);
m.put(cacheName, c);
}
this.caches = Collections.unmodifiableMap(m);
}

StartupLoggingUtils.checkRequestLogging();

hostName = cfg.getNodeName();
Expand Down Expand Up @@ -1265,6 +1289,17 @@ public void shutdown() {
// Now clear all the cores that are being operated upon.
solrCores.close();

final Map<String, SolrCache<?, ?>> closeCaches = caches;
if (closeCaches != null) {
for (Map.Entry<String, SolrCache<?, ?>> e : caches.entrySet()) {
try {
e.getValue().close();
} catch (Exception ex) {
log.warn("error closing node-level cache: {}", e.getKey(), ex);
}
}
}

objectCache.clear();

// It's still possible that one of the pending dynamic load operation is waiting, so wake it
Expand Down
16 changes: 16 additions & 0 deletions solr/core/src/java/org/apache/solr/core/NodeConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.logging.DeprecationLog;
import org.apache.solr.logging.LogWatcherConfig;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.ModuleUtils;
Expand Down Expand Up @@ -109,6 +110,8 @@ public class NodeConfig {

private final MetricsConfig metricsConfig;

private final Map<String, CacheConfig> cachesConfig;

private final PluginInfo tracerConfig;

// Track if this config was loaded from zookeeper so that we can skip validating the zookeeper
Expand Down Expand Up @@ -144,6 +147,7 @@ private NodeConfig(
Properties solrProperties,
PluginInfo[] backupRepositoryPlugins,
MetricsConfig metricsConfig,
Map<String, CacheConfig> cachesConfig,
PluginInfo tracerConfig,
boolean fromZookeeper,
String defaultZkHost,
Expand Down Expand Up @@ -179,6 +183,7 @@ private NodeConfig(
this.solrProperties = solrProperties;
this.backupRepositoryPlugins = backupRepositoryPlugins;
this.metricsConfig = metricsConfig;
this.cachesConfig = cachesConfig == null ? Collections.emptyMap() : cachesConfig;
this.tracerConfig = tracerConfig;
this.fromZookeeper = fromZookeeper;
this.defaultZkHost = defaultZkHost;
Expand Down Expand Up @@ -395,6 +400,10 @@ public MetricsConfig getMetricsConfig() {
return metricsConfig;
}

public Map<String, CacheConfig> getCachesConfig() {
return cachesConfig;
}

public PluginInfo getTracerConfiguratorPluginInfo() {
return tracerConfig;
}
Expand Down Expand Up @@ -600,6 +609,7 @@ public static class NodeConfigBuilder {
private Properties solrProperties = new Properties();
private PluginInfo[] backupRepositoryPlugins;
private MetricsConfig metricsConfig;
private Map<String, CacheConfig> cachesConfig;
private PluginInfo tracerConfig;
private boolean fromZookeeper = false;
private String defaultZkHost;
Expand Down Expand Up @@ -769,6 +779,11 @@ public NodeConfigBuilder setMetricsConfig(MetricsConfig metricsConfig) {
return this;
}

public NodeConfigBuilder setCachesConfig(Map<String, CacheConfig> cachesConfig) {
this.cachesConfig = cachesConfig;
return this;
}

public NodeConfigBuilder setTracerConfig(PluginInfo tracerConfig) {
this.tracerConfig = tracerConfig;
return this;
Expand Down Expand Up @@ -881,6 +896,7 @@ public NodeConfig build() {
solrProperties,
backupRepositoryPlugins,
metricsConfig,
cachesConfig,
tracerConfig,
fromZookeeper,
defaultZkHost,
Expand Down
3 changes: 2 additions & 1 deletion solr/core/src/java/org/apache/solr/core/SolrConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,8 @@ private SolrConfig(
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);

Map<String, CacheConfig> userCacheConfigs =
CacheConfig.getMultipleConfigs(this, "query/cache", get("query").getAll("cache"));
CacheConfig.getMultipleConfigs(
getResourceLoader(), this, "query/cache", get("query").getAll("cache"));
List<PluginInfo> caches = getPluginInfos(SolrCache.class.getName());
if (!caches.isEmpty()) {
for (PluginInfo c : caches) {
Expand Down
1 change: 1 addition & 0 deletions solr/core/src/java/org/apache/solr/core/SolrCore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2663,6 +2663,7 @@ public RefCounted<SolrIndexSearcher> getSearcher(
future =
searcherExecutor.submit(
() -> {
newSearcher.bootstrapFirstSearcher();
for (SolrEventListener listener : firstSearcherListeners) {
try {
listener.newSearcher(newSearcher, null);
Expand Down
16 changes: 16 additions & 0 deletions solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.solr.common.util.Utils;
import org.apache.solr.logging.LogWatcherConfig;
import org.apache.solr.metrics.reporters.SolrJmxReporter;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.DOMConfigNode;
Expand Down Expand Up @@ -177,6 +178,7 @@ public static NodeConfig fromConfig(
// Remove this line in 10.0
configBuilder.setHiddenSysProps(getHiddenSysProps(root.get("metrics")));
configBuilder.setMetricsConfig(getMetricsConfig(root.get("metrics")));
configBuilder.setCachesConfig(getCachesConfig(loader, root.get("caches")));
configBuilder.setFromZookeeper(fromZookeeper);
configBuilder.setDefaultZkHost(defaultZkHost);
configBuilder.setCoreAdminHandlerActions(coreAdminHandlerActions);
Expand Down Expand Up @@ -693,6 +695,20 @@ private static MetricsConfig getMetricsConfig(ConfigNode metrics) {
return builder.setMetricReporterPlugins(reporterPlugins).build();
}

private static Map<String, CacheConfig> getCachesConfig(
SolrResourceLoader loader, ConfigNode caches) {
Map<String, CacheConfig> ret =
CacheConfig.getMultipleConfigs(loader, null, null, caches.getAll("cache"));
for (CacheConfig c : ret.values()) {
if (c.getRegenerator() != null) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"node-level caches should not be configured with a regenerator!");
}
}
return Collections.unmodifiableMap(ret);
}

private static Object decodeNullValue(Object o) {
if (o instanceof String) { // check if it's a JSON object
String str = (String) o;
Expand Down
14 changes: 11 additions & 3 deletions solr/core/src/java/org/apache/solr/search/CacheConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,13 @@ public void setRegenerator(CacheRegenerator regenerator) {
}

public static Map<String, CacheConfig> getMultipleConfigs(
SolrConfig solrConfig, String configPath, List<ConfigNode> nodes) {
SolrResourceLoader loader, SolrConfig solrConfig, String configPath, List<ConfigNode> nodes) {
if (nodes == null || nodes.size() == 0) return new LinkedHashMap<>();
Map<String, CacheConfig> result = CollectionUtil.newHashMap(nodes.size());
for (ConfigNode node : nodes) {
if (node.boolAttr("enabled", true)) {
CacheConfig config =
getConfig(solrConfig, node.name(), node.attributes().asMap(), configPath);
getConfig(loader, solrConfig, node.name(), node.attributes().asMap(), configPath);
result.put(config.args.get(NAME), config);
}
}
Expand All @@ -108,6 +108,15 @@ public static CacheConfig getConfig(SolrConfig solrConfig, ConfigNode node, Stri

public static CacheConfig getConfig(
SolrConfig solrConfig, String nodeName, Map<String, String> attrs, String xpath) {
return getConfig(solrConfig.getResourceLoader(), solrConfig, nodeName, attrs, xpath);
}

public static CacheConfig getConfig(
SolrResourceLoader loader,
SolrConfig solrConfig,
String nodeName,
Map<String, String> attrs,
String xpath) {
CacheConfig config = new CacheConfig();
config.nodeName = nodeName;
Map<String, String> attrsCopy = CollectionUtil.newLinkedHashMap(attrs.size());
Expand All @@ -131,7 +140,6 @@ public static CacheConfig getConfig(
config.args.put(NAME, config.nodeName);
}

SolrResourceLoader loader = solrConfig.getResourceLoader();
config.cacheImpl = config.args.get("class");
if (config.cacheImpl == null) config.cacheImpl = "solr.CaffeineCache";
config.clazz =
Expand Down
6 changes: 6 additions & 0 deletions solr/core/src/java/org/apache/solr/search/CaffeineCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,12 @@ public void setMaxRamMB(int maxRamMB) {
}
}

protected void adjustMetrics(long hitsAdjust, long insertsAdjust, long lookupsAdjust) {
hits.add(-hitsAdjust);
inserts.add(-insertsAdjust);
lookups.add(-lookupsAdjust);
}

@Override
public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old) {
if (regenerator == null) {
Expand Down
10 changes: 10 additions & 0 deletions solr/core/src/java/org/apache/solr/search/SolrCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,16 @@ public enum State {
*/
public State getState();

/**
* A hook for caches that would like to perform some initialization for the first registered
* searcher. This method is analogous to {@link #warm(SolrIndexSearcher, SolrCache)}. The default
* implementation is a no-op. Implementers should not retain object references to the specified
* searcher.
*/
default void initialSearcher(SolrIndexSearcher initialSearcher) {
// no-op
}

/**
* Warm this cache associated with <code>searcher</code> using the <code>old</code> cache object.
* <code>this</code> and <code>old</code> will have the same concrete type.
Expand Down
12 changes: 12 additions & 0 deletions solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2430,6 +2430,18 @@ public boolean intersects(DocSet a, DocsEnumState deState) throws IOException {
return a.intersects(getDocSet(deState));
}

/**
* Called on the initial searcher for each core, immediately before <code>firstSearcherListeners
* </code> are called for the searcher. This provides the opportunity to perform initialization on
* the first registered searcher before the searcher begins to see any <code>firstSearcher</code>
* -triggered events.
*/
public void bootstrapFirstSearcher() {
for (SolrCache<?, ?> solrCache : cacheList) {
solrCache.initialSearcher(this);
}
}

/** Warm this searcher based on an old one (primarily for auto-cache warming). */
@SuppressWarnings({"unchecked"})
public void warm(SolrIndexSearcher old) {
Expand Down
34 changes: 34 additions & 0 deletions solr/core/src/test-files/solr/collection1/conf/solrconfig-nodelevelcaches.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
<?xml version="1.0" ?>

<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
<requestHandler name="/select" class="solr.SearchHandler" />
<query>
<filterCache
class="solr.ThinCache"
parentCacheName="myNodeLevelCacheThin"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

myNodeLevelCacheThin needs to define in solr.xml?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, exactly.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the motivation to keep this separate (not in solrconfig)? Would be good to have cacheref element in colrconfig ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the motivation to keep this separate (not in solrconfig)?

The cache is configured in the config file that's associated with the same scope as the cache itself. solrconfig.xml configures properties associated with the collection/core. A node-level cache is configured in the node-level config file, solr.xml.

What is the motivation to keep this separate (not in solrconfig)? Would be good to have cacheref element in colrconfig ?

I'm not sure I understand?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like both should be configure in one place ? probably its easier to track

size="5"
initialSize="5"/>
</query>
</config>
30 changes: 30 additions & 0 deletions solr/core/src/test-files/solr/solr-nodelevelcaches.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<solr>
<caches>
<cache name="myNodeLevelCache"
size="10"
initialSize="10"
/>
<cache name="myNodeLevelCacheThin"
class="solr.ThinCache$NodeLevelCache"
size="10"
initialSize="10"
/>
</caches>
</solr>
16 changes: 16 additions & 0 deletions solr/core/src/test/org/apache/solr/core/TestSolrXml.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.search.CacheConfig;
import org.apache.solr.search.CaffeineCache;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.ThinCache;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.hamcrest.MatcherAssert;
import org.junit.Before;
Expand Down Expand Up @@ -152,6 +156,18 @@ public void testPropertySub() throws IOException {
assertFalse("schema cache", cfg.hasSchemaCache());
}

public void testNodeLevelCache() throws IOException {
Path testSrcRoot = TEST_PATH();
Files.copy(testSrcRoot.resolve("solr-nodelevelcaches.xml"), solrHome.resolve("solr.xml"));

NodeConfig cfg = SolrXmlConfig.fromSolrHome(solrHome, new Properties());
Map<String, CacheConfig> cachesConfig = cfg.getCachesConfig();
SolrCache<?, ?> nodeLevelCache = cachesConfig.get("myNodeLevelCache").newInstance(null);
assertTrue(nodeLevelCache instanceof CaffeineCache);
SolrCache<?, ?> nodeLevelCacheThin = cachesConfig.get("myNodeLevelCacheThin").newInstance(null);
assertTrue(nodeLevelCacheThin instanceof ThinCache.NodeLevelCache);
}

public void testExplicitNullGivesDefaults() {
System.setProperty("jetty.port", "8000");
String solrXml =
Expand Down
Loading
Loading