Skip to content

Commit

Permalink
Refactor access to VM options and move some VM options to oal.util.Co…
Browse files Browse the repository at this point in the history
…nstants (#12754)
  • Loading branch information
uschindler committed Nov 3, 2023
1 parent 82ee701 commit 71b3e4c
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 133 deletions.
3 changes: 3 additions & 0 deletions lucene/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ Improvements

* GITHUB#12689: TaskExecutor to cancel all tasks on exception to avoid needless computation. (Luca Cavanna)

* GITHUB#12754: Refactor lookup of Hotspot VM options and do not initialize constants with NULL
if SecurityManager prevents access. (Uwe Schindler)

Optimizations
---------------------
* GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import java.lang.StackWalker.StackFrame;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
Expand All @@ -31,7 +29,7 @@
import java.util.function.Predicate;
import java.util.logging.Logger;
import java.util.stream.Stream;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.VectorUtil;

/**
Expand Down Expand Up @@ -129,7 +127,7 @@ static VectorizationProvider lookup(boolean testMode) {
"Vector bitsize and/or integer vectors enforcement; using default vectorization provider outside of testMode");
return new DefaultVectorizationProvider();
}
if (isClientVM()) {
if (Constants.IS_CLIENT_VM) {
LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled");
return new DefaultVectorizationProvider();
}
Expand Down Expand Up @@ -188,23 +186,6 @@ private static boolean isAffectedByJDK8301190() {
&& !Objects.equals("I", "i".toUpperCase(Locale.getDefault()));
}

@SuppressWarnings("removal")
@SuppressForbidden(reason = "security manager")
private static boolean isClientVM() {
try {
final PrivilegedAction<Boolean> action =
() -> System.getProperty("java.vm.info", "").contains("emulated-client");
return AccessController.doPrivileged(action);
} catch (
@SuppressWarnings("unused")
SecurityException e) {
LOG.warning(
"SecurityManager denies permission to 'java.vm.info' system property, so state of C2 compiler can't be detected. "
+ "In case of performance issues allow access to this property.");
return false;
}
}

// add all possible callers here as FQCN:
private static final Set<String> VALID_CALLERS = Set.of("org.apache.lucene.util.VectorUtil");

Expand Down
82 changes: 60 additions & 22 deletions lucene/core/src/java/org/apache/lucene/util/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,22 @@
*/
package org.apache.lucene.util;

import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Objects;
import java.util.logging.Logger;

/** Some useful constants. */
public final class Constants {
private Constants() {} // can't construct

private static final String UNKNOWN = "Unknown";

/** JVM vendor info. */
public static final String JVM_VENDOR = System.getProperty("java.vm.vendor");
public static final String JVM_VENDOR = getSysProp("java.vm.vendor", UNKNOWN);

/** JVM vendor name. */
public static final String JVM_NAME = System.getProperty("java.vm.name");
public static final String JVM_NAME = getSysProp("java.vm.name", UNKNOWN);

/**
* Get the full version string of the current runtime.
Expand All @@ -51,7 +58,7 @@ private Constants() {} // can't construct
@Deprecated public static final String JAVA_VERSION = System.getProperty("java.version");

/** The value of <code>System.getProperty("os.name")</code>. * */
public static final String OS_NAME = System.getProperty("os.name");
public static final String OS_NAME = getSysProp("os.name", UNKNOWN);

/** True iff running on Linux. */
public static final boolean LINUX = OS_NAME.startsWith("Linux");
Expand All @@ -69,37 +76,68 @@ private Constants() {} // can't construct
public static final boolean FREE_BSD = OS_NAME.startsWith("FreeBSD");

/** The value of <code>System.getProperty("os.arch")</code>. */
public static final String OS_ARCH = System.getProperty("os.arch");
public static final String OS_ARCH = getSysProp("os.arch", UNKNOWN);

/** The value of <code>System.getProperty("os.version")</code>. */
public static final String OS_VERSION = System.getProperty("os.version");
public static final String OS_VERSION = getSysProp("os.version", UNKNOWN);

/** The value of <code>System.getProperty("java.vendor")</code>. */
public static final String JAVA_VENDOR = System.getProperty("java.vendor");
public static final String JAVA_VENDOR = getSysProp("java.vendor", UNKNOWN);

/** True iff the Java runtime is a client runtime and C2 compiler is not enabled */
public static final boolean IS_CLIENT_VM =
getSysProp("java.vm.info", "").contains("emulated-client");

/** True iff running on a 64bit JVM */
public static final boolean JRE_IS_64BIT;
public static final boolean JRE_IS_64BIT = is64Bit();

/** true iff we know fast FMA is supported, to deliver less error */
public static final boolean HAS_FAST_FMA =
(IS_CLIENT_VM == false)
&& Objects.equals(OS_ARCH, "amd64")
&& HotspotVMOptions.get("UseFMA").map(Boolean::valueOf).orElse(false);

private static boolean is64Bit() {
final String datamodel = getSysProp("sun.arch.data.model");
if (datamodel != null) {
return datamodel.contains("64");
} else {
return (OS_ARCH != null && OS_ARCH.contains("64"));
}
}

static {
boolean is64Bit = false;
String datamodel = null;
private static String getSysProp(String property) {
try {
datamodel = System.getProperty("sun.arch.data.model");
if (datamodel != null) {
is64Bit = datamodel.contains("64");
}
return doPrivileged(() -> System.getProperty(property));
} catch (
@SuppressWarnings("unused")
SecurityException ex) {
SecurityException se) {
logSecurityWarning(property);
return null;
}
if (datamodel == null) {
if (OS_ARCH != null && OS_ARCH.contains("64")) {
is64Bit = true;
} else {
is64Bit = false;
}
}

private static String getSysProp(String property, String def) {
try {
return doPrivileged(() -> System.getProperty(property, def));
} catch (
@SuppressWarnings("unused")
SecurityException se) {
logSecurityWarning(property);
return def;
}
JRE_IS_64BIT = is64Bit;
}

private static void logSecurityWarning(String property) {
var log = Logger.getLogger(Constants.class.getName());
log.warning("SecurityManager prevented access to system property: " + property);
}

// Extracted to a method to be able to apply the SuppressForbidden annotation
@SuppressWarnings("removal")
@SuppressForbidden(reason = "security manager")
private static <T> T doPrivileged(PrivilegedAction<T> action) {
return AccessController.doPrivileged(action);
}

/**
Expand Down
90 changes: 90 additions & 0 deletions lucene/core/src/java/org/apache/lucene/util/HotspotVMOptions.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;

import java.lang.reflect.Method;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.logging.Logger;

/** Accessor to get Hotspot VM Options (if available). */
final class HotspotVMOptions {
private HotspotVMOptions() {} // can't construct

/** True if the Java VM is based on Hotspot and has the Hotspot MX bean readable by Lucene */
public static final boolean IS_HOTSPOT;

/**
* Returns an optional with the value of a Hotspot VM option. If the VM option does not exist or
* is not readable, returns an empty optional.
*/
public static Optional<String> get(String name) {
return ACCESSOR.apply(Objects.requireNonNull(name, "name"));
}

private static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";
private static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";
private static final Function<String, Optional<String>> ACCESSOR;

static {
boolean isHotspot = false;
Function<String, Optional<String>> accessor = name -> Optional.empty();
try {
final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);
// we use reflection for this, because the management factory is not part
// of java.base module:
final Object hotSpotBean =
Class.forName(MANAGEMENT_FACTORY_CLASS)
.getMethod("getPlatformMXBean", Class.class)
.invoke(null, beanClazz);
if (hotSpotBean != null) {
final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
final Method getValueMethod = getVMOptionMethod.getReturnType().getMethod("getValue");
isHotspot = true;
accessor =
name -> {
try {
final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, name);
return Optional.of(getValueMethod.invoke(vmOption).toString());
} catch (@SuppressWarnings("unused")
ReflectiveOperationException
| RuntimeException e) {
return Optional.empty();
}
};
}
} catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
isHotspot = false;
final Logger log = Logger.getLogger(HotspotVMOptions.class.getName());
final Module module = HotspotVMOptions.class.getModule();
final ModuleLayer layer = module.getLayer();
// classpath / unnamed module has no layer, so we need to check:
if (layer != null
&& layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {
log.warning(
"Lucene cannot access JVM internals to optimize algorithms or calculate object sizes, unless the 'jdk.management' Java module "
+ "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor].");
} else {
log.warning(
"Lucene cannot optimize algorithms or calculate object sizes for JVMs that are not based on Hotspot or a compatible implementation.");
}
}
IS_HOTSPOT = isHotspot;
ACCESSOR = accessor;
}
}
62 changes: 6 additions & 56 deletions lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.security.AccessControlException;
import java.security.AccessController;
Expand All @@ -30,7 +29,6 @@
import java.util.IdentityHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.logging.Logger;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
Expand Down Expand Up @@ -112,64 +110,16 @@ private RamUsageEstimator() {}
/** For testing only */
static final boolean JVM_IS_HOTSPOT_64BIT;

static final String MANAGEMENT_FACTORY_CLASS = "java.lang.management.ManagementFactory";
static final String HOTSPOT_BEAN_CLASS = "com.sun.management.HotSpotDiagnosticMXBean";

/** Initialize constants and try to collect information about the JVM internals. */
static {
if (Constants.JRE_IS_64BIT) {
if (Constants.JRE_IS_64BIT && HotspotVMOptions.IS_HOTSPOT) {
// Try to get compressed oops and object alignment (the default seems to be 8 on Hotspot);
// (this only works on 64 bit, on 32 bits the alignment and reference size is fixed):
boolean compressedOops = false;
int objectAlignment = 8;
boolean isHotspot = false;
try {
final Class<?> beanClazz = Class.forName(HOTSPOT_BEAN_CLASS);
// we use reflection for this, because the management factory is not part
// of Java 8's compact profile:
final Object hotSpotBean =
Class.forName(MANAGEMENT_FACTORY_CLASS)
.getMethod("getPlatformMXBean", Class.class)
.invoke(null, beanClazz);
if (hotSpotBean != null) {
isHotspot = true;
final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
try {
final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "UseCompressedOops");
compressedOops =
Boolean.parseBoolean(
vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());
} catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
isHotspot = false;
}
try {
final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
objectAlignment =
Integer.parseInt(
vmOption.getClass().getMethod("getValue").invoke(vmOption).toString());
} catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
isHotspot = false;
}
}
} catch (@SuppressWarnings("unused") ReflectiveOperationException | RuntimeException e) {
isHotspot = false;
final Logger log = Logger.getLogger(RamUsageEstimator.class.getName());
final Module module = RamUsageEstimator.class.getModule();
final ModuleLayer layer = module.getLayer();
// classpath / unnamed module has no layer, so we need to check:
if (layer != null
&& layer.findModule("jdk.management").map(module::canRead).orElse(false) == false) {
log.warning(
"Lucene cannot correctly calculate object sizes on 64bit JVMs, unless the 'jdk.management' Java module "
+ "is readable [please add 'jdk.management' to modular application either by command line or its module descriptor]");
} else {
log.warning(
"Lucene cannot correctly calculate object sizes on 64bit JVMs that are not based on Hotspot or a compatible implementation.");
}
}
JVM_IS_HOTSPOT_64BIT = isHotspot;
COMPRESSED_REFS_ENABLED = compressedOops;
NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
JVM_IS_HOTSPOT_64BIT = true;
COMPRESSED_REFS_ENABLED =
HotspotVMOptions.get("UseCompressedOops").map(Boolean::valueOf).orElse(false);
NUM_BYTES_OBJECT_ALIGNMENT =
HotspotVMOptions.get("ObjectAlignmentInBytes").map(Integer::valueOf).orElse(8);
// reference size is 4, if we have compressed oops:
NUM_BYTES_OBJECT_REF = COMPRESSED_REFS_ENABLED ? 4 : 8;
// "best guess" based on reference size:
Expand Down
Loading

0 comments on commit 71b3e4c

Please sign in to comment.