From 8efd720e5503e1a01ac17b9e6f2929b4683bd7d6 Mon Sep 17 00:00:00 2001 From: Tim Owen Date: Thu, 18 Jan 2018 14:20:06 +0000 Subject: [PATCH 1/3] Allow ConcurrentMergeScheduler AutoIOThrottle to be configured via solrconfig.xml --- .../apache/lucene/index/ConcurrentMergeScheduler.java | 2 +- .../java/org/apache/solr/update/SolrIndexConfig.java | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java index 57dc0c3ad7f2..6e930c48a204 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java +++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java @@ -117,7 +117,7 @@ public class ConcurrentMergeScheduler extends MergeScheduler { protected double targetMBPerSec = START_MB_PER_SEC; /** true if we should rate-limit writes for each merge */ - private boolean doAutoIOThrottle = false; + private boolean doAutoIOThrottle = true; private double forceMergeMBPerSec = Double.POSITIVE_INFINITY; diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index d484e85b13e1..d68d9eb286b4 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -381,7 +381,16 @@ private MergeScheduler buildMergeScheduler(IndexSchema schema) { if (maxThreadCount == null) { maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount(); } - ((ConcurrentMergeScheduler)scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount); + ConcurrentMergeScheduler cmScheduler = (ConcurrentMergeScheduler)scheduler; + cmScheduler.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); + Boolean autoIOThrottle = (Boolean) args.remove("autoIOThrottle"); + if (autoIOThrottle != null) { + if (autoIOThrottle) { + cmScheduler.enableAutoIOThrottle(); + } else { + cmScheduler.disableAutoIOThrottle(); + } + } SolrPluginUtils.invokeSetters(scheduler, args); } else { SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs); From 98ffa4be712d2edf640c4231e06adc2d270d5c30 Mon Sep 17 00:00:00 2001 From: Tim Owen Date: Thu, 18 Jan 2018 14:36:07 +0000 Subject: [PATCH 2/3] Support setting a fixed write rate throttle for ConcurrentMergeScheduler --- .../index/ConcurrentMergeScheduler.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java index 6e930c48a204..84fccdf31eda 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java +++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java @@ -121,6 +121,9 @@ public class ConcurrentMergeScheduler extends MergeScheduler { private double forceMergeMBPerSec = Double.POSITIVE_INFINITY; + /** Write throttle rate when doAutoIOThrottle is false */ + private double fixedMergeMBPerSec = Double.POSITIVE_INFINITY; + /** Sole constructor, with all settings set to default * values. */ public ConcurrentMergeScheduler() { @@ -224,13 +227,23 @@ public synchronized boolean getAutoIOThrottle() { return doAutoIOThrottle; } + /** Set the per-merge IO throttle rate when auto IO throttling is disabled (default: {@code Double.POSITIVE_INFINITY}). */ + public synchronized void setFixedMergeMBPerSec(double v) { + fixedMergeMBPerSec = v; + } + + /** Get the per-merge IO throttle rate when auto IO throttling is disabled. */ + public synchronized double getFixedMergeMBPerSec() { + return fixedMergeMBPerSec; + } + /** Returns the currently set per-merge IO writes rate limit, if {@link #enableAutoIOThrottle} - * was called, else {@code Double.POSITIVE_INFINITY}. */ + * was called, else {@link #getFixedMergeMBPerSec}. */ public synchronized double getIORateLimitMBPerSec() { if (doAutoIOThrottle) { return targetMBPerSec; } else { - return Double.POSITIVE_INFINITY; + return fixedMergeMBPerSec; } } @@ -354,10 +367,10 @@ protected synchronized void updateMergeThreads() { } else if (merge.maxNumSegments != -1) { newMBPerSec = forceMergeMBPerSec; } else if (doAutoIOThrottle == false) { - newMBPerSec = Double.POSITIVE_INFINITY; + newMBPerSec = fixedMergeMBPerSec; } else if (merge.estimatedMergeBytes < MIN_BIG_MERGE_MB*1024*1024) { // Don't rate limit small merges: - newMBPerSec = Double.POSITIVE_INFINITY; + newMBPerSec = fixedMergeMBPerSec; } else { newMBPerSec = targetMBPerSec; } @@ -385,7 +398,7 @@ protected synchronized void updateMergeThreads() { if (newMBPerSec == 0.0) { message.append(" now stop"); } else if (curMBPerSec == 0.0) { - if (newMBPerSec == Double.POSITIVE_INFINITY) { + if (newMBPerSec == fixedMergeMBPerSec) { message.append(" now resume"); } else { message.append(String.format(Locale.ROOT, " now resume to %.1f MB/sec", newMBPerSec)); @@ -427,10 +440,10 @@ private synchronized void initDynamicDefaults(IndexWriter writer) throws IOExcep } } - private static String rateToString(double mbPerSec) { + private String rateToString(double mbPerSec) { if (mbPerSec == 0.0) { return "stopped"; - } else if (mbPerSec == Double.POSITIVE_INFINITY) { + } else if (mbPerSec == fixedMergeMBPerSec) { return "unlimited"; } else { return String.format(Locale.ROOT, "%.1f MB/sec", mbPerSec); From f1d6f54e2655f12c4f624a870cebb53fbc90d6aa Mon Sep 17 00:00:00 2001 From: Tim Owen Date: Thu, 18 Jan 2018 16:54:28 +0000 Subject: [PATCH 3/3] Common up some downcasts --- .../src/java/org/apache/solr/update/SolrIndexConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index d68d9eb286b4..b91e1675b858 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -372,16 +372,16 @@ private MergeScheduler buildMergeScheduler(IndexSchema schema) { // LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads // if someone has them configured. if (scheduler instanceof ConcurrentMergeScheduler) { + ConcurrentMergeScheduler cmScheduler = (ConcurrentMergeScheduler) scheduler; NamedList args = mergeSchedulerInfo.initArgs.clone(); Integer maxMergeCount = (Integer) args.remove("maxMergeCount"); if (maxMergeCount == null) { - maxMergeCount = ((ConcurrentMergeScheduler) scheduler).getMaxMergeCount(); + maxMergeCount = cmScheduler.getMaxMergeCount(); } Integer maxThreadCount = (Integer) args.remove("maxThreadCount"); if (maxThreadCount == null) { - maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount(); + maxThreadCount = cmScheduler.getMaxThreadCount(); } - ConcurrentMergeScheduler cmScheduler = (ConcurrentMergeScheduler)scheduler; cmScheduler.setMaxMergesAndThreads(maxMergeCount, maxThreadCount); Boolean autoIOThrottle = (Boolean) args.remove("autoIOThrottle"); if (autoIOThrottle != null) {