From 30f643079b97c007fe96547eeed798e78e3df83c Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 12 Sep 2023 10:02:17 -0400 Subject: [PATCH 1/2] SOLR-16966: Add a first-class cache for OrdinalMaps backport unmerged upstream PR at commit 9eeb7e0d47684357dc3b13a7ea34b2498c995138 --- .../java/org/apache/solr/core/SolrConfig.java | 13 +- .../index/SlowCompositeReaderWrapper.java | 77 ++-- .../org/apache/solr/search/CaffeineCache.java | 2 +- .../apache/solr/search/OrdMapRegenerator.java | 264 +++++++++++++ .../apache/solr/search/SolrIndexSearcher.java | 12 +- .../solr/search/facet/UnInvertedField.java | 3 +- .../solr/search/function/OrdFieldSource.java | 4 +- .../function/ReverseOrdFieldSource.java | 4 +- .../org/apache/solr/update/VersionInfo.java | 4 +- .../index/TestSlowCompositeReaderWrapper.java | 11 +- .../apache/solr/schema/TestPointFields.java | 3 +- .../apache/solr/search/TestOrdMapCache.java | 361 ++++++++++++++++++ .../solr/uninverting/TestDocTermOrds.java | 12 +- .../TestDocTermOrdsUninvertLimit.java | 3 +- .../solr/uninverting/TestFieldCache.java | 3 +- .../TestFieldCacheVsDocValues.java | 3 +- .../uninverting/TestLegacyFieldCache.java | 5 +- .../uninverting/TestUninvertingReader.java | 6 +- .../pages/caches-warming.adoc | 28 ++ 19 files changed, 753 insertions(+), 65 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/search/OrdMapRegenerator.java create mode 100644 solr/core/src/test/org/apache/solr/search/TestOrdMapCache.java diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 75a60ccfae8..24ac37784aa 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -78,6 +78,7 @@ import org.apache.solr.schema.IndexSchemaFactory; import org.apache.solr.search.CacheConfig; import org.apache.solr.search.CaffeineCache; +import org.apache.solr.search.OrdMapRegenerator; import org.apache.solr.search.QParserPlugin; import org.apache.solr.search.SolrCache; import org.apache.solr.search.ValueSourceParser; @@ -306,6 +307,8 @@ private SolrConfig( // filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32); // filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f); + updateHandlerInfo = loadUpdatehandlerInfo(); // must do this before configuring ordMapCache + useFilterForSortedQuery = get("query").get("useFilterForSortedQuery").boolVal(false); queryResultWindowSize = Math.max(1, get("query").get("queryResultWindowSize").intVal(1)); queryResultMaxDocsCached = @@ -329,6 +332,13 @@ private SolrConfig( conf = new CacheConfig(CaffeineCache.class, args, null); } fieldValueCacheConfig = conf; + conf = CacheConfig.getConfig(this, get("query").get("ordMapCache"), "query/ordMapCache"); + if (conf != null) { + OrdMapRegenerator.configureRegenerator(this, conf); + ordMapCacheConfig = conf; + } else { + ordMapCacheConfig = OrdMapRegenerator.getDefaultCacheConfig(this); + } useColdSearcher = get("query").get("useColdSearcher").boolVal(false); dataDir = get("dataDir").txt(); if (dataDir != null && dataDir.length() == 0) dataDir = null; @@ -356,8 +366,6 @@ private SolrConfig( } this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); - updateHandlerInfo = loadUpdatehandlerInfo(); - final var requestParsersNode = get("requestDispatcher").get("requestParsers"); multipartUploadLimitKB = @@ -675,6 +683,7 @@ public SolrRequestParsers getRequestParsers() { // public final int filtOptCacheSize; // public final float filtOptThreshold; // SolrIndexSearcher - caches configurations + public final CacheConfig ordMapCacheConfig; public final CacheConfig filterCacheConfig; public final CacheConfig queryResultCacheConfig; public final CacheConfig documentCacheConfig; diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java index b066ba373fd..c45caa863ce 100644 --- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java +++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java @@ -18,9 +18,6 @@ import java.io.IOException; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.CompositeReader; @@ -54,6 +51,11 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.Version; import org.apache.lucene.util.packed.PackedInts; +import org.apache.solr.search.CaffeineCache; +import org.apache.solr.search.OrdMapRegenerator; +import org.apache.solr.search.OrdMapRegenerator.OrdinalMapValue; +import org.apache.solr.search.SolrCache; +import org.apache.solr.util.IOFunction; /** * This class forces a composite reader (eg a {@link MultiReader} or {@link DirectoryReader}) to @@ -76,24 +78,34 @@ public final class SlowCompositeReaderWrapper extends LeafReader { // also have a cached FieldInfos instance so this is consistent. SOLR-12878 private final FieldInfos fieldInfos; - // TODO: this could really be a weak map somewhere else on the coreCacheKey, - // but do we really need to optimize slow-wrapper any more? - final Map cachedOrdMaps = new ConcurrentHashMap<>(); + public static final SolrCache NO_CACHED_ORDMAPS = + new CaffeineCache<>() { + @Override + public OrdinalMapValue computeIfAbsent( + String key, IOFunction mappingFunction) + throws IOException { + return mappingFunction.apply(key); + } + }; + + final SolrCache cachedOrdMaps; /** * This method is sugar for getting an {@link LeafReader} from an {@link IndexReader} of any kind. * If the reader is already atomic, it is returned unchanged, otherwise wrapped by this class. */ - public static LeafReader wrap(IndexReader reader) throws IOException { + public static LeafReader wrap(IndexReader reader, SolrCache ordMapCache) + throws IOException { if (reader instanceof CompositeReader) { - return new SlowCompositeReaderWrapper((CompositeReader) reader); + return new SlowCompositeReaderWrapper((CompositeReader) reader, ordMapCache); } else { assert reader instanceof LeafReader; return (LeafReader) reader; } } - SlowCompositeReaderWrapper(CompositeReader reader) throws IOException { + SlowCompositeReaderWrapper( + CompositeReader reader, SolrCache cachedOrdMaps) throws IOException { in = reader; in.registerParentReader(this); if (reader.leaves().isEmpty()) { @@ -114,6 +126,7 @@ public static LeafReader wrap(IndexReader reader) throws IOException { metaData = new LeafMetaData(createdVersionMajor, minVersion, null); } fieldInfos = FieldInfos.getMergedFieldInfos(in); + this.cachedOrdMaps = cachedOrdMaps; } @Override @@ -203,28 +216,19 @@ public SortedDocValues getSortedDocValues(String field) throws IOException { return null; } - // at this point in time we are able to formulate the producer - OrdinalMap map = null; CacheHelper cacheHelper = getReaderCacheHelper(); - Function producer = - (notUsed) -> { - try { - OrdinalMap mapping = - OrdinalMap.build( - cacheHelper == null ? null : cacheHelper.getKey(), values, PackedInts.DEFAULT); - return mapping; - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - // either we use a cached result that gets produced eventually during caching, // or we produce directly without caching + OrdinalMap map; if (cacheHelper != null) { - map = cachedOrdMaps.computeIfAbsent(field + cacheHelper.getKey(), producer); + IOFunction producer = + (notUsed) -> + OrdMapRegenerator.wrapValue( + OrdinalMap.build(cacheHelper.getKey(), values, PackedInts.DEFAULT)); + map = cachedOrdMaps.computeIfAbsent(field, producer).get(); } else { - map = producer.apply("notUsed"); + map = OrdinalMap.build(null, values, PackedInts.DEFAULT); } return new MultiSortedDocValues(values, starts, map, totalCost); @@ -275,28 +279,19 @@ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException return null; } - // at this point in time we are able to formulate the producer - OrdinalMap map = null; CacheHelper cacheHelper = getReaderCacheHelper(); - Function producer = - (notUsed) -> { - try { - OrdinalMap mapping = - OrdinalMap.build( - cacheHelper == null ? null : cacheHelper.getKey(), values, PackedInts.DEFAULT); - return mapping; - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - // either we use a cached result that gets produced eventually during caching, // or we produce directly without caching + OrdinalMap map; if (cacheHelper != null) { - map = cachedOrdMaps.computeIfAbsent(field + cacheHelper.getKey(), producer); + IOFunction producer = + (notUsed) -> + OrdMapRegenerator.wrapValue( + OrdinalMap.build(cacheHelper.getKey(), values, PackedInts.DEFAULT)); + map = cachedOrdMaps.computeIfAbsent(field, producer).get(); } else { - map = producer.apply("notUsed"); + map = OrdinalMap.build(null, values, PackedInts.DEFAULT); } return new MultiDocValues.MultiSortedSetDocValues(values, starts, map, totalCost); diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java index 99e3ae65330..2d98fdb37d2 100644 --- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java +++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java @@ -165,7 +165,7 @@ private Cache buildCache(Cache prev) { builder.maximumWeight(maxRamBytes); builder.weigher( (k, v) -> (int) (RamUsageEstimator.sizeOfObject(k) + RamUsageEstimator.sizeOfObject(v))); - } else { + } else if (maxSize < Integer.MAX_VALUE) { builder.maximumSize(maxSize); } Cache newCache; diff --git a/solr/core/src/java/org/apache/solr/search/OrdMapRegenerator.java b/solr/core/src/java/org/apache/solr/search/OrdMapRegenerator.java new file mode 100644 index 00000000000..34891b031f4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/OrdMapRegenerator.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import static org.apache.solr.common.params.CommonParams.NAME; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.OrdinalMap; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.packed.PackedInts; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.util.IOFunction; + +/** Cache regenerator that builds OrdinalMap instances against the new searcher. */ +public class OrdMapRegenerator implements CacheRegenerator { + + private static final long DEFAULT_REGEN_KEEPALIVE_NANOS = TimeUnit.MINUTES.toNanos(2); + private static final OrdMapRegenerator DEFAULT_INSTANCE = + new OrdMapRegenerator(DEFAULT_REGEN_KEEPALIVE_NANOS); + + private final long regenKeepAliveNanos; + + public OrdMapRegenerator() { + this(DEFAULT_REGEN_KEEPALIVE_NANOS); + // default ctor in case someone specifies this class via standard `"regen"=[className]` syntax + } + + private OrdMapRegenerator(long regenKeepAliveNanos) { + this.regenKeepAliveNanos = regenKeepAliveNanos; + } + + public static class OrdinalMapValue implements Supplier { + private final OrdinalMap ordinalMap; + private long accessTimestampNanos; + + private OrdinalMapValue(OrdinalMap ordinalMap, long accessTimestampNanos) { + this.ordinalMap = ordinalMap; + this.accessTimestampNanos = accessTimestampNanos; + } + + @Override + public OrdinalMap get() { + accessTimestampNanos = System.nanoTime(); + return ordinalMap; + } + } + + public static OrdinalMapValue wrapValue(OrdinalMap ordinalMap) { + return new OrdinalMapValue(ordinalMap, 0); + } + + public static CacheConfig getDefaultCacheConfig(SolrConfig solrConfig) { + // for back-compat, default to an effectively unlimited-sized cache with no regeneration + Map args = new HashMap<>(); + args.put(NAME, "ordMapCache"); + args.put("size", Integer.toString(Integer.MAX_VALUE)); // effectively unlimited + args.put("initialSize", "10"); + CacheConfig c = new CacheConfig(CaffeineCache.class, args, null); + configureRegenerator(solrConfig, c); + return c; + } + + public static void configureRegenerator(SolrConfig solrConfig, CacheConfig config) { + if (config.getRegenerator() != null) { + return; + } + String keepAliveConfig = (String) config.toMap(Collections.emptyMap()).get("regenKeepAlive"); + final long regenKeepAliveNanos; + if (keepAliveConfig == null || keepAliveConfig.isEmpty()) { + long osiNanos; + if (solrConfig == null || (osiNanos = getOpenSearcherIntervalNanos(solrConfig)) == -1) { + regenKeepAliveNanos = DEFAULT_REGEN_KEEPALIVE_NANOS; + } else { + regenKeepAliveNanos = osiNanos << 1; + } + } else { + int lastIdx = keepAliveConfig.length() - 1; + String sub = keepAliveConfig.substring(0, lastIdx); + switch (keepAliveConfig.charAt(lastIdx)) { + case 's': + regenKeepAliveNanos = TimeUnit.SECONDS.toNanos(Long.parseLong(sub)); + break; + case 'm': + regenKeepAliveNanos = TimeUnit.MINUTES.toNanos(Long.parseLong(sub)); + break; + case 'h': + regenKeepAliveNanos = TimeUnit.HOURS.toNanos(Long.parseLong(sub)); + break; + case 'd': + regenKeepAliveNanos = TimeUnit.DAYS.toNanos(Long.parseLong(sub)); + break; + case '%': + int keepAlivePct = Integer.parseInt(sub); + if (keepAlivePct < 0) { + throw new IllegalArgumentException( + "regenKeepAlive % must be positive; found " + keepAlivePct); + } + long osiNanos; + if (solrConfig == null || (osiNanos = getOpenSearcherIntervalNanos(solrConfig)) == -1) { + throw new IllegalArgumentException( + "regenKeepAlive % must only be configured in conjunction with autoCommit time"); + } else { + regenKeepAliveNanos = (osiNanos * keepAlivePct) / 100; + } + break; + default: + regenKeepAliveNanos = TimeUnit.MILLISECONDS.toNanos(Long.parseLong(keepAliveConfig)); + break; + } + } + if (regenKeepAliveNanos == DEFAULT_REGEN_KEEPALIVE_NANOS) { + config.setRegenerator(DEFAULT_INSTANCE); + return; + } + config.setRegenerator(new OrdMapRegenerator(regenKeepAliveNanos)); + } + + private static long getOpenSearcherIntervalNanos(SolrConfig solrConfig) { + SolrConfig.UpdateHandlerInfo uinfo = solrConfig.getUpdateHandlerInfo(); + if (uinfo == null) { + return -1; + } else if (uinfo.autoSoftCommmitMaxTime != -1) { + if (uinfo.openSearcher && uinfo.autoCommmitMaxTime != -1) { + return TimeUnit.MILLISECONDS.toNanos( + Math.min(uinfo.autoCommmitMaxTime, uinfo.autoSoftCommmitMaxTime)); + } else { + return TimeUnit.MILLISECONDS.toNanos(uinfo.autoSoftCommmitMaxTime); + } + } else if (uinfo.openSearcher && uinfo.autoCommmitMaxTime != -1) { + return TimeUnit.MILLISECONDS.toNanos(uinfo.autoCommmitMaxTime); + } else { + return -1; + } + } + + @Override + public boolean regenerateItem( + SolrIndexSearcher newSearcher, + SolrCache newCache, + SolrCache oldCache, + K oldKey, + V oldVal) + throws IOException { + DirectoryReader in = newSearcher.getIndexReader(); + IndexReader.CacheHelper cacheHelper = in.getReaderCacheHelper(); + if (cacheHelper == null) { + return false; + } + + final List leaves = in.leaves(); + final int size = leaves.size(); + + if (size < 2) { + // we don't need OrdinalMaps for these trivial cases + return false; + } + + final long extantTimestamp = ((OrdinalMapValue) oldVal).accessTimestampNanos; + if (System.nanoTime() - extantTimestamp > regenKeepAliveNanos) { + // it has been long enough since this was last accessed that we don't want to carry it forward + return true; + } + + final String field = (String) oldKey; + boolean anyReal = false; + final SortedDocValues[] sdvs = new SortedDocValues[size]; + final SortedSetDocValues[] ssdvs = new SortedSetDocValues[size]; + DocValuesType type = null; + for (int i = 0; i < size; i++) { + LeafReaderContext context = leaves.get(i); + final LeafReader reader = context.reader(); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); + if (fieldInfo == null) { + sdvs[i] = DocValues.emptySorted(); + ssdvs[i] = DocValues.emptySortedSet(); + } else { + DocValuesType leafType = fieldInfo.getDocValuesType(); + if (type == null) { + type = leafType; + } else if (leafType != type) { + throw new IllegalStateException("mixed docValues types " + type + " and " + leafType); + } + switch (leafType) { + case SORTED: + SortedDocValues sdv = reader.getSortedDocValues(field); + if (sdv == null) { + sdv = DocValues.emptySorted(); + } else { + anyReal = true; + } + sdvs[i] = sdv; + break; + case SORTED_SET: + SortedSetDocValues ssdv = reader.getSortedSetDocValues(field); + if (ssdv == null) { + ssdv = DocValues.emptySortedSet(); + } else { + anyReal = true; + } + ssdvs[i] = ssdv; + break; + default: + throw new IllegalStateException("unexpected docValues type: " + leafType); + } + } + } + if (!anyReal) { + // All empty for this field, but should still warm others + return true; + } + + IndexReader.CacheKey readerKey = cacheHelper.getKey(); + final IOFunction> producer; + switch (type) { + case SORTED: + producer = + (notUsed) -> + new OrdinalMapValue( + OrdinalMap.build(readerKey, sdvs, PackedInts.DEFAULT), extantTimestamp); + break; + case SORTED_SET: + producer = + (notUsed) -> + new OrdinalMapValue( + OrdinalMap.build(readerKey, ssdvs, PackedInts.DEFAULT), extantTimestamp); + break; + default: + throw new IllegalStateException(); + } + + @SuppressWarnings("unchecked") + SolrCache> c = (SolrCache>) newCache; + c.computeIfAbsent(field, producer); + return true; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 0dba978c4d8..5fede2ff853 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -107,6 +107,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.OrdMapRegenerator.OrdinalMapValue; import org.apache.solr.search.facet.UnInvertedField; import org.apache.solr.search.stats.StatsCache; import org.apache.solr.search.stats.StatsSource; @@ -156,6 +157,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI private final boolean cachingEnabled; private final SolrCache filterCache; + private final SolrCache ordMapCache; private final SolrCache queryResultCache; private final SolrCache fieldValueCache; private final LongAdder fullSortCount = new LongAdder(); @@ -341,7 +343,6 @@ public SolrIndexSearcher( this.directoryFactory = directoryFactory; this.reader = (DirectoryReader) super.readerContext.reader(); this.rawReader = r; - this.leafReader = SlowCompositeReaderWrapper.wrap(this.reader); this.core = core; this.statsCache = core.createStatsCache(); this.schema = schema; @@ -375,11 +376,16 @@ public SolrIndexSearcher( this.queryResultMaxDocsCached = solrConfig.queryResultMaxDocsCached; this.useFilterForSortedQuery = solrConfig.useFilterForSortedQuery; + ordMapCache = solrConfig.ordMapCacheConfig.newInstance(); + assert ordMapCache != null; + this.leafReader = SlowCompositeReaderWrapper.wrap(this.reader, ordMapCache); + this.docFetcher = new SolrDocumentFetcher(this, solrConfig, enableCache); this.cachingEnabled = enableCache; if (cachingEnabled) { final ArrayList clist = new ArrayList<>(); + clist.add(ordMapCache); fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null @@ -618,6 +624,10 @@ public Iterable getFieldNames() { .collect(Collectors.toUnmodifiableList()); } + public SolrCache getOrdMapCache() { + return ordMapCache; + } + public SolrCache getFilterCache() { return filterCache; } diff --git a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java index 5ddd4705116..08b8cd2bb49 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java @@ -202,7 +202,8 @@ public UnInvertedField(String field, SolrIndexSearcher searcher) throws IOExcept // TODO: it's wasteful to create one of these each time // but DocTermOrds will throw an exception if it thinks the field has doc values (which is // faked by UnInvertingReader) - LeafReader r = SlowCompositeReaderWrapper.wrap(searcher.getRawReader()); + LeafReader r = + SlowCompositeReaderWrapper.wrap(searcher.getRawReader(), searcher.getOrdMapCache()); uninvert(r, r.getLiveDocs(), prefix == null ? null : new BytesRef(prefix)); } catch (IllegalStateException ise) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ise); diff --git a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java index bbcb81363b5..87aeb937e81 100644 --- a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java @@ -93,14 +93,14 @@ public FunctionValues getValues(Map context, LeafReaderContext r for (LeafReaderContext raw : leaves) { hidingLeaves[upto++] = NumericHidingLeafReader.wrap(raw.reader(), field); } - r = SlowCompositeReaderWrapper.wrap(new MultiReader(hidingLeaves)); + r = SlowCompositeReaderWrapper.wrap(new MultiReader(hidingLeaves), is.getOrdMapCache()); } else { // reuse ordinalmap r = ((SolrIndexSearcher) o).getSlowAtomicReader(); } } else { IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader(); - r = SlowCompositeReaderWrapper.wrap(topReader); + r = SlowCompositeReaderWrapper.wrap(topReader, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); } // if it's e.g. tokenized/multivalued, emulate old behavior of single-valued fc final SortedDocValues sindex = diff --git a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java index 5e725f79dab..a8c41f1c6e8 100644 --- a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java @@ -91,14 +91,14 @@ public FunctionValues getValues(Map context, LeafReaderContext r for (LeafReaderContext raw : leaves) { hidingLeaves[upto++] = NumericHidingLeafReader.wrap(raw.reader(), field); } - r = SlowCompositeReaderWrapper.wrap(new MultiReader(hidingLeaves)); + r = SlowCompositeReaderWrapper.wrap(new MultiReader(hidingLeaves), is.getOrdMapCache()); } else { // reuse ordinalmap r = ((SolrIndexSearcher) o).getSlowAtomicReader(); } } else { IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader(); - r = SlowCompositeReaderWrapper.wrap(topReader); + r = SlowCompositeReaderWrapper.wrap(topReader, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); } // if it's e.g. tokenized/multivalued, emulate old behavior of single-valued fc final SortedDocValues sindex = diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index 139873fdc96..a0c3908ae74 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -297,7 +297,9 @@ private long getMaxVersionFromIndexedTerms(IndexSearcher searcher) throws IOExce assert !versionField.getType().isPointField(); final String versionFieldName = versionField.getName(); - final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()); + final LeafReader leafReader = + SlowCompositeReaderWrapper.wrap( + searcher.getIndexReader(), SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); final Terms versionTerms = leafReader.terms(versionFieldName); final Long max = (versionTerms != null) ? LegacyNumericUtils.getMaxLong(versionTerms) : null; if (null != max) { diff --git a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java index bc6b699f1bf..c0961d691d4 100644 --- a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java +++ b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java @@ -34,6 +34,9 @@ import org.apache.lucene.tests.util.TestUtil; import org.apache.lucene.util.BytesRef; import org.apache.solr.SolrTestCase; +import org.apache.solr.search.OrdMapRegenerator; +import org.apache.solr.search.OrdMapRegenerator.OrdinalMapValue; +import org.apache.solr.search.SolrCache; public class TestSlowCompositeReaderWrapper extends SolrTestCase { @@ -50,7 +53,8 @@ public void testCoreListenerOnSlowCompositeReaderWrapper() throws IOException { w.close(); final IndexReader reader = DirectoryReader.open(w.w.getDirectory()); - final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader); + final LeafReader leafReader = + SlowCompositeReaderWrapper.wrap(reader, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); final int numListeners = TestUtil.nextInt(random(), 1, 10); final List listeners = new ArrayList<>(); @@ -114,8 +118,11 @@ public void testOrdMapsAreCached() throws Exception { w.addDocument(doc); IndexReader reader = w.getReader(); assertTrue(reader.leaves().size() > 1); + @SuppressWarnings("unchecked") + SolrCache ordMapCache = + OrdMapRegenerator.getDefaultCacheConfig(null).newInstance(); SlowCompositeReaderWrapper slowWrapper = - (SlowCompositeReaderWrapper) SlowCompositeReaderWrapper.wrap(reader); + (SlowCompositeReaderWrapper) SlowCompositeReaderWrapper.wrap(reader, ordMapCache); assertEquals(0, slowWrapper.cachedOrdMaps.size()); assertEquals(MultiSortedDocValues.class, slowWrapper.getSortedDocValues("sorted").getClass()); assertEquals(1, slowWrapper.cachedOrdMaps.size()); diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java index b1e7e6b8162..82387a067e7 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java +++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java @@ -5751,7 +5751,8 @@ private void doTestInternals(String field, String[] values) throws IOException { // our own SlowCompositeReader to check DocValues on disk w/o the UninvertingReader // added by SolrIndexSearcher final LeafReader leafReaderForCheckingDVs = - SlowCompositeReaderWrapper.wrap(searcher.getRawReader()); + SlowCompositeReaderWrapper.wrap( + searcher.getRawReader(), searcher.getOrdMapCache()); if (sf.indexed()) { assertEquals( diff --git a/solr/core/src/test/org/apache/solr/search/TestOrdMapCache.java b/solr/core/src/test/org/apache/solr/search/TestOrdMapCache.java new file mode 100644 index 00000000000..42880a31d8f --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/TestOrdMapCache.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.util.EmbeddedSolrServerTestRule; +import org.apache.solr.util.SolrClientTestRule; +import org.junit.ClassRule; + +public class TestOrdMapCache extends SolrTestCaseJ4 { + + @ClassRule + public static final SolrClientTestRule solrClientTestRule = + new EmbeddedSolrServerTestRule() { + @Override + protected void before() throws Throwable { + startSolr(LuceneTestCase.createTempDir()); + } + }; + + private static final String ORD_MAP_CACHE_METRIC_NAME = "CACHE.searcher.ordMapCache"; + + private static Map getOrdMapCacheMetrics(SolrClient client, String collectionName) { + @SuppressWarnings("unchecked") + SolrMetricManager.GaugeWrapper> metrics = + (SolrMetricManager.GaugeWrapper>) + ((EmbeddedSolrServer) client) + .getCoreContainer() + .getMetricManager() + .getMetrics( + "solr.core.".concat(collectionName), + (metricName, metric) -> ORD_MAP_CACHE_METRIC_NAME.equals(metricName)) + .get(ORD_MAP_CACHE_METRIC_NAME); + return metrics.getValue(); + } + + public void testDefaultBehavior() throws Exception { + String name = "default"; + Path configSet = LuceneTestCase.createTempDir(); + SolrTestCaseJ4.copyMinConf(configSet.toFile()); + + solrClientTestRule.newCollection(name).withConfigSet(configSet.toString()).create(); + + SolrClient client = solrClientTestRule.getSolrClient(name); + Map metrics; + + client.add(sdoc("id", "1")); + client.commit(); + + // initial sanity-check + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // would trigger ordmap build + + // for 1-seg index, no ordmap will actually be built + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.add(sdoc("id", "2")); + client.commit(); + + client.query(params("q", "*:*", "facet", "true", "facet.field", "id")); // trigger ordmap build + + // for 2-seg index, ordmap will actually be built + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("lookups")); + assertEquals(1L, metrics.get("inserts")); + assertEquals(0L, metrics.get("hits")); + + client.add(sdoc("id", "3")); + client.commit(); + + // expect ordmap to not be auto-warmed + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(0, metrics.get("size")); + assertEquals(1L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(0L, metrics.get("cumulative_hits")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // will get auto-warmed ordmap + + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(2L, metrics.get("cumulative_lookups")); + assertEquals(2L, metrics.get("cumulative_inserts")); + assertEquals(0L, metrics.get("cumulative_hits")); + } + + public void testLongKeepAlive() throws Exception { + String name = "longKeepAlive"; + String keepAlive; + switch (random().nextInt(5)) { + case 0: + keepAlive = "60000"; + break; + case 1: + keepAlive = "60s"; + break; + case 2: + keepAlive = "1m"; + break; + case 3: + keepAlive = "1h"; + break; + case 4: + keepAlive = "1d"; + break; + default: + throw new IllegalStateException(); + } + keepAlive = "regenKeepAlive=\"" + keepAlive + "\"\n"; + Path configSet = LuceneTestCase.createTempDir(); + SolrTestCaseJ4.copyMinConf(configSet.toFile()); + // insert a special ordMapCache configuration + Path solrConfig = configSet.resolve("conf/solrconfig.xml"); + Files.writeString( + solrConfig, + Files.readString(solrConfig) + .replace( + "", + "\n" + + "\n" + + "")); + + solrClientTestRule.newCollection(name).withConfigSet(configSet.toString()).create(); + + SolrClient client = solrClientTestRule.getSolrClient(name); + Map metrics; + + client.add(sdoc("id", "1")); + client.commit(); + + // initial sanity-check + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // would trigger ordmap build + + // for 1-seg index, no ordmap will actually be built + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.add(sdoc("id", "2")); + client.commit(); + + client.query(params("q", "*:*", "facet", "true", "facet.field", "id")); // trigger ordmap build + + // for 2-seg index, ordmap will actually be built + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("lookups")); + assertEquals(1L, metrics.get("inserts")); + assertEquals(0L, metrics.get("hits")); + + client.add(sdoc("id", "3")); + client.commit(); + + // expect ordmap to be auto-warmed + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(0L, metrics.get("cumulative_hits")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // will get auto-warmed ordmap + + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(2L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(1L, metrics.get("cumulative_hits")); + } + + public void testShortKeepAlive() throws Exception { + String name = "shortKeepAlive"; + Path configSet = LuceneTestCase.createTempDir(); + SolrTestCaseJ4.copyMinConf(configSet.toFile()); + // insert a special ordMapCache configuration + Path solrConfig = configSet.resolve("conf/solrconfig.xml"); + Files.writeString( + solrConfig, + Files.readString(solrConfig) + .replace( + "", + "\n" + + "\n" + + "")); + + solrClientTestRule.newCollection(name).withConfigSet(configSet.toString()).create(); + + SolrClient client = solrClientTestRule.getSolrClient(name); + Map metrics; + + client.add(sdoc("id", "1")); + client.commit(); + + // initial sanity-check + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // would trigger ordmap build + + // for 1-seg index, no ordmap will actually be built + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.add(sdoc("id", "2")); + client.commit(); + + client.query(params("q", "*:*", "facet", "true", "facet.field", "id")); // trigger ordmap build + + // for 2-seg index, ordmap will actually be built + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("lookups")); + assertEquals(1L, metrics.get("inserts")); + assertEquals(0L, metrics.get("hits")); + + client.add(sdoc("id", "3")); + client.commit(); + + // expect ordmap to be auto-warmed + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(0L, metrics.get("cumulative_hits")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // will get auto-warmed ordmap + + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(2L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(1L, metrics.get("cumulative_hits")); + + Thread.sleep(300); + + client.add(sdoc("id", "4")); + client.commit(); + + // entry should still be auto-warmed at this point + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(2L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(1L, metrics.get("cumulative_hits")); + + // wait long enough that the entry should no longer be auto-warmed + Thread.sleep(300); + + client.add(sdoc("id", "5")); + client.commit(); + + // we waited beyond keepAlive; expect ordmap to be empty + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(0, metrics.get("size")); + assertEquals(2L, metrics.get("cumulative_lookups")); + assertEquals(1L, metrics.get("cumulative_inserts")); + assertEquals(1L, metrics.get("cumulative_hits")); + + client.query(params("q", "*:*", "facet", "true", "facet.field", "id")); // will re-build ordmap + + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(3L, metrics.get("cumulative_lookups")); + assertEquals(2L, metrics.get("cumulative_inserts")); + assertEquals(1L, metrics.get("cumulative_hits")); + } + + public void testSizeLimited() throws Exception { + String name = "sizeLimited"; + Path configSet = LuceneTestCase.createTempDir(); + SolrTestCaseJ4.copyMinConf(configSet.toFile()); + // insert a special ordMapCache configuration + Path solrConfig = configSet.resolve("conf/solrconfig.xml"); + Files.writeString( + solrConfig, + Files.readString(solrConfig) + .replace( + "", + "\n" + + "\n" + + "")); + + solrClientTestRule.newCollection(name).withConfigSet(configSet.toString()).create(); + + SolrClient client = solrClientTestRule.getSolrClient(name); + Map metrics; + + client.add(sdoc("id", "1", "other", "1")); + client.commit(); + + // initial sanity-check + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "id")); // would trigger ordmap build + + // for 1-seg index, no ordmap will actually be built + assertEquals(0, getOrdMapCacheMetrics(client, name).get("size")); + + client.add(sdoc("id", "2", "other", "2")); + client.commit(); + + client.query(params("q", "*:*", "facet", "true", "facet.field", "id")); // trigger ordmap build + + // for 2-seg index, ordmap will actually be built + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(1L, metrics.get("lookups")); + assertEquals(1L, metrics.get("inserts")); + assertEquals(0L, metrics.get("hits")); + + client.query( + params("q", "*:*", "facet", "true", "facet.field", "other")); // trigger ordmap build/evict + + // no actual hits, just evict/insert (thrashing) + metrics = getOrdMapCacheMetrics(client, name); + assertEquals(1, metrics.get("size")); + assertEquals(2L, metrics.get("lookups")); + assertEquals(2L, metrics.get("inserts")); + assertEquals(0L, metrics.get("hits")); + } +} diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java index f955196312a..b70284094ac 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java @@ -69,7 +69,8 @@ public void testEmptyIndex() throws IOException { final DirectoryReader ir = DirectoryReader.open(dir); TestUtil.checkReader(ir); - final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir); + final LeafReader composite = + SlowCompositeReaderWrapper.wrap(ir, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(composite); // check the leaves @@ -114,7 +115,8 @@ public void testSimple() throws Exception { final IndexReader r = w.getReader(); w.close(); - final LeafReader ar = SlowCompositeReaderWrapper.wrap(r); + final LeafReader ar = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(ar); final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); SortedSetDocValues iter = dto.iterator(ar); @@ -218,7 +220,8 @@ public void testRandom() throws Exception { if (VERBOSE) { System.out.println("TEST: top reader"); } - LeafReader slowR = SlowCompositeReaderWrapper.wrap(r); + LeafReader slowR = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(slowR); verify(slowR, idToOrds, termsArray, null); @@ -307,7 +310,8 @@ public void testRandomWithPrefix() throws Exception { System.out.println("TEST: reader=" + r); } - LeafReader slowR = SlowCompositeReaderWrapper.wrap(r); + LeafReader slowR = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(slowR); for (String prefix : prefixesArray) { diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java index 380be298978..ce580cb8232 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java @@ -83,7 +83,8 @@ public void testTriggerUnInvertLimit() throws IOException { final IndexReader r = DirectoryReader.open(dir); try { - final LeafReader ar = SlowCompositeReaderWrapper.wrap(r); + final LeafReader ar = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(ar); final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off if (SHOULD_TRIGGER) { diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java index efbfa28b268..fadaa4914c1 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java @@ -304,7 +304,8 @@ public void testEmptyIndex() throws Exception { dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500)); writer.close(); IndexReader r = DirectoryReader.open(dir); - LeafReader reader = SlowCompositeReaderWrapper.wrap(r); + LeafReader reader = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(reader); FieldCache.DEFAULT.getTerms(reader, "foobar"); FieldCache.DEFAULT.getTermsIndex(reader, "foobar"); diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java index 89ae4181f85..d6a66bba15a 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java @@ -170,7 +170,8 @@ public void testHugeBinaryValues() throws Exception { DirectoryReader r = DirectoryReader.open(w); - try (LeafReader ar = SlowCompositeReaderWrapper.wrap(r)) { + try (LeafReader ar = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS)) { TestUtil.checkReader(ar); BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field"); diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java index b1750a59921..b9a9aeb0b27 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java @@ -91,7 +91,7 @@ public static void beforeClass() throws Exception { writer.addDocument(doc); } IndexReader r = writer.getReader(); - reader = SlowCompositeReaderWrapper.wrap(r); + reader = SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(reader); writer.close(); } @@ -177,7 +177,8 @@ public void testEmptyIndex() throws Exception { dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500)); writer.close(); IndexReader r = DirectoryReader.open(dir); - LeafReader reader = SlowCompositeReaderWrapper.wrap(r); + LeafReader reader = + SlowCompositeReaderWrapper.wrap(r, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(reader); FieldCache.DEFAULT.getTerms(reader, "foobar"); FieldCache.DEFAULT.getTermsIndex(reader, "foobar"); diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java index 9137ad3cd62..ce0e8e6fa90 100644 --- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java +++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java @@ -310,7 +310,8 @@ public void testSortedSetIntegerManyValues() throws IOException { } // check the composite of all leaves: exact expectation of set size - final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir); + final LeafReader composite = + SlowCompositeReaderWrapper.wrap(ir, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(composite); for (String f : MULTI_VALUES) { @@ -339,7 +340,8 @@ public void testSortedSetEmptyIndex() throws IOException { final DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), UNINVERT_MAP); TestUtil.checkReader(ir); - final LeafReader composite = SlowCompositeReaderWrapper.wrap(ir); + final LeafReader composite = + SlowCompositeReaderWrapper.wrap(ir, SlowCompositeReaderWrapper.NO_CACHED_ORDMAPS); TestUtil.checkReader(composite); for (String f : UNINVERT_MAP.keySet()) { diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc index 4a8e584f125..be9f0a36c8c 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc @@ -135,6 +135,34 @@ The filter cache is a good candidate for enabling `async` computation. ---- +=== OrdMap Cache + +This is a low-level cache that holds mappings between global ordinals and per-segment ordinals for string docValues. +It is an essential component of functionality like faceting and join queries, and an implicit default instance of this cache will be configured if no explicit configuration is provided. + +The `ordMapCache` usually contains relatively few values -- at most one value is created per field per searcher, and values are created on-demand in response to an operation that requires global<=>segment ordinal mappings for a given field. + +The behavior of the implicit default cache is equivalent to legacy behavior, which opaquely cached these values in an unbounded map, newly created for each searcher, in response to queries. In fact, indirect warming of the cached ordinal maps is often one of the most important functions of `filterCache` warming queries. + +Manually configuring the `ordMapCache` provides finer grained control, including the ability to: +1. place constraints on resource usage (max size or bytes) +2. directly autowarm the `ordMapCache` + +`ordMapCache` is configured similarly to other caches, but with the additional config parameter `regenKeepAlive`. During autowarming, cache entry metadata is used to avoid auto-warming `ordMapCache` entries that have not been accessed within a specified amount of time. This is important for `ordMapCache`, because due to its often-unlimited enforced size, and its often-quite-small size in practice, entries will not be replaced in the same way that they would be for other caches that have higher turnover. `regenKeepAlive` solves this by allowing entries to "age out" if they are unused. +`regenKeepAlive` may be specified in milliseconds, seconds, minutes, hours, or days (e.g., `60000`, `60s`, `1m`, `1h`, `1d`). `regenKeepAlive` may also be specified as a percentage of the effective autoCommit interval. + +As a low-level cache, `ordMapCache` is warmed before other caches. + +[source,xml] +---- + +---- + + === Query Result Cache The `queryResultCache` holds the results of previous searches: ordered lists of document IDs (DocList) based on a query, a sort, and the range of documents requested. From d9be388123159e959717be5ae4ed94d054c803cc Mon Sep 17 00:00:00 2001 From: Michael Gibney Date: Tue, 12 Sep 2023 10:21:56 -0400 Subject: [PATCH 2/2] updates to accommodate FS custom CacheConfig ctor signature --- .../core/src/java/org/apache/solr/search/SolrIndexSearcher.java | 2 +- .../org/apache/solr/index/TestSlowCompositeReaderWrapper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 5fede2ff853..04ed9e7f8da 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -376,7 +376,7 @@ public SolrIndexSearcher( this.queryResultMaxDocsCached = solrConfig.queryResultMaxDocsCached; this.useFilterForSortedQuery = solrConfig.useFilterForSortedQuery; - ordMapCache = solrConfig.ordMapCacheConfig.newInstance(); + ordMapCache = solrConfig.ordMapCacheConfig.newInstance(core); assert ordMapCache != null; this.leafReader = SlowCompositeReaderWrapper.wrap(this.reader, ordMapCache); diff --git a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java index c0961d691d4..96ece3823ab 100644 --- a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java +++ b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java @@ -120,7 +120,7 @@ public void testOrdMapsAreCached() throws Exception { assertTrue(reader.leaves().size() > 1); @SuppressWarnings("unchecked") SolrCache ordMapCache = - OrdMapRegenerator.getDefaultCacheConfig(null).newInstance(); + OrdMapRegenerator.getDefaultCacheConfig(null).newInstance(null); SlowCompositeReaderWrapper slowWrapper = (SlowCompositeReaderWrapper) SlowCompositeReaderWrapper.wrap(reader, ordMapCache); assertEquals(0, slowWrapper.cachedOrdMaps.size());