Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support explicit reconciliation of heap directory size account against filesystem #143

Merged
merged 1 commit into from
Sep 14, 2023
Merged

Support explicit reconciliation of heap directory size account against filesystem #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM;
import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM;

import java.io.Closeable;
import java.io.File;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
Expand Down Expand Up @@ -116,6 +117,27 @@ public CoreAdminHandler(final CoreContainer coreContainer) {
this.coreAdminAsyncTracker = new CoreAdminAsyncTracker();
}

private static final ThreadLocal<Integer> RECONCILE_THRESHOLD = new ThreadLocal<>();

private static final Closeable CLEAR_RECONCILE_THRESHOLD = RECONCILE_THRESHOLD::remove;

public static Closeable setReconcileThreshold(String spec) {
if (spec != null && !spec.isEmpty()) {
int size = Integer.parseInt(spec);
if (size >= 0) {
RECONCILE_THRESHOLD.set(size);
} else if (size == -1) {
// support special value `-1` to effectively just print log messages
RECONCILE_THRESHOLD.set(Integer.MAX_VALUE);
}
}
return CLEAR_RECONCILE_THRESHOLD;
}

public static Integer getReconcileThreshold() {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this actually be used somewhere or am I missing something?

Copy link
Collaborator Author

@magibney magibney Sep 14, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's used from a plugin (the SizeAware Directory impl). So yes, not in anything that's a part of this PR!

return RECONCILE_THRESHOLD.get();
}

@Override
public final void init(NamedList<?> args) {
throw new SolrException(
Expand Down
37 changes: 21 additions & 16 deletions solr/core/src/java/org/apache/solr/handler/admin/StatusOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.solr.handler.admin;

import java.io.Closeable;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand All @@ -29,36 +30,40 @@

class StatusOp implements CoreAdminHandler.CoreAdminOp {
@Override
@SuppressWarnings("try")
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
SolrParams params = it.req.getParams();

String cname = params.get(CoreAdminParams.CORE);
String indexInfo = params.get(CoreAdminParams.INDEX_INFO);
String reconcile = params.get(CoreAdminParams.RECONCILE_THRESHOLD);
boolean isIndexInfoNeeded = Boolean.parseBoolean(null == indexInfo ? "true" : indexInfo);
NamedList<Object> status = new SimpleOrderedMap<>();
Map<String, Exception> failures = new HashMap<>();
for (Map.Entry<String, CoreContainer.CoreLoadFailure> failure :
it.handler.coreContainer.getCoreInitFailures().entrySet()) {
failures.put(failure.getKey(), failure.getValue().exception);
}
if (cname == null) {
List<String> nameList = it.handler.coreContainer.getAllCoreNames();
nameList.sort(null);
for (String name : nameList) {
try (Closeable c = CoreAdminHandler.setReconcileThreshold(reconcile)) {
if (cname == null) {
List<String> nameList = it.handler.coreContainer.getAllCoreNames();
nameList.sort(null);
for (String name : nameList) {
status.add(
name,
CoreAdminOperation.getCoreStatus(it.handler.coreContainer, name, isIndexInfoNeeded));
}
it.rsp.add("initFailures", failures);
} else {
failures =
failures.containsKey(cname)
? Collections.singletonMap(cname, failures.get(cname))
: Collections.<String, Exception>emptyMap();
it.rsp.add("initFailures", failures);
status.add(
name,
CoreAdminOperation.getCoreStatus(it.handler.coreContainer, name, isIndexInfoNeeded));
cname,
CoreAdminOperation.getCoreStatus(it.handler.coreContainer, cname, isIndexInfoNeeded));
}
it.rsp.add("initFailures", failures);
} else {
failures =
failures.containsKey(cname)
? Collections.singletonMap(cname, failures.get(cname))
: Collections.<String, Exception>emptyMap();
it.rsp.add("initFailures", failures);
status.add(
cname,
CoreAdminOperation.getCoreStatus(it.handler.coreContainer, cname, isIndexInfoNeeded));
}
it.rsp.add("status", status);
}
Expand Down
7 changes: 7 additions & 0 deletions solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ public abstract class CoreAdminParams {
/** Should the STATUS request include index info * */
public static final String INDEX_INFO = "indexInfo";

/**
* For STATUS indexInfo requests that track Directory size on-heap, this arg causes the on-heap
* size to be reconciled with the filesystem, re-initializing the on-heap tracking if the
* difference in bytes is more than the specified amount.
*/
public static final String RECONCILE_THRESHOLD = "reconcileThreshold";

/** If you rename something, what is the new name * */
public static final String NAME = "name";

Expand Down
Loading