Skip to content

Commit

Permalink
An eye-gouging way to limit suppressAccessChecks to just the three JA…
Browse files Browse the repository at this point in the history 
…Rs that need them. (#13164)
  • Loading branch information
@dweiss
dweiss authored Mar 8, 2024
1 parent 49db29e commit 1c77e23
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 211 deletions.
38 changes: 33 additions & 5 deletions gradle/testing/randomization.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,18 @@ allprojects {
// Resolve test option values after all evaluation is complete.
allprojects {
plugins.withType(JavaPlugin) {
configurations {
secManagerExclusions
}
dependencies {
secManagerExclusions ( "com.carrotsearch.randomizedtesting:randomizedtesting-runner", {
exclude group: "junit"
})
secManagerExclusions ( "junit:junit", {
exclude group: "org.hamcrest"
})
}

afterEvaluate {
ext.testOptionsResolved = testOptions.findAll { opt ->
propertyOrDefault(opt.propName, opt.value) != null
Expand Down Expand Up @@ -155,7 +167,7 @@ allprojects {
}

// Append resolved test properties to the test task.
tasks.withType(Test) { task ->
tasks.withType(Test) { Test task ->
// TODO: we could remove some options that are only relevant to the build environment
// and not the test JVM itself.
systemProperties testOptionsResolved
Expand All @@ -172,13 +184,13 @@ allprojects {

// Enable security manager, if requested. We could move the selection of security manager and security policy
// to each project's build/ configuration but it seems compact enough to keep it here for now.
def securityArgumentProvider = new SecurityArgumentProvider(commonDir: project(":lucene").layout.projectDirectory)
def securityArgumentProvider = new SecurityArgumentProvider()
securityArgumentProvider.commonDir = project(":lucene").layout.projectDirectory
securityArgumentProvider.otherProperties = project.objects.mapProperty(String, String)

if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
if (project.path.endsWith(".tests")) {
// LUCENE-10301: for now, do not use the security manager for modular tests (test framework is not available).
} else if (project.path == ":lucene:replicator") {
systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/replicator-tests.policy")
} else if (project.path.startsWith(":lucene")) {
systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/tests.policy")
Expand All @@ -189,6 +201,14 @@ allprojects {
systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/')
systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString()
systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString()

securityArgumentProvider.otherProperties.put("randomizedtesting.jar", project.provider { ->
return configurations.secManagerExclusions.resolve().find { it.name.startsWith("randomizedtesting-runner-") }.absolutePath
})
securityArgumentProvider.otherProperties.put("junit.jar", project.provider { ->
return configurations.secManagerExclusions.resolve().find { it.name.startsWith("junit-") }.absolutePath
})
securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)
}

doFirst {
Expand Down Expand Up @@ -242,12 +262,20 @@ class SecurityArgumentProvider implements CommandLineArgumentProvider {
@PathSensitive(PathSensitivity.RELATIVE)
RegularFile javaSecurityPolicy

@Input
MapProperty<String, String> otherProperties

@Override
Iterable<String> asArguments() {
def args = ["-Dcommon.dir=${commonDir.getAsFile()}"]
if (javaSecurityPolicy) {
args.add("-Djava.security.policy=${javaSecurityPolicy.getAsFile()}")
}

otherProperties.getOrElse(Map.of()).forEach { key, value ->
args.add("-D${key}=${value}")
}

return args
}
}
94 changes: 0 additions & 94 deletions gradle/testing/randomization/policies/replicator-tests.policy
View file

This file was deleted.

33 changes: 16 additions & 17 deletions gradle/testing/randomization/policies/tests.policy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,34 +33,29 @@ grant {

// misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created
permission java.nio.file.LinkPermission "hard";
// needed by SSD detection tests in TestIOUtils (creates symlinks)
permission java.nio.file.LinkPermission "symbolic";

// needed by randomizedtesting runner to identify test methods.
// needed by RamUsageEstimator, and randomizedtesting runner to identify test methods.
permission java.lang.RuntimePermission "accessDeclaredMembers";
// needed by certain tests to capture sysout/syserr:
permission java.lang.RuntimePermission "setIO";
// needed by randomized runner to catch failures from other threads:
// needed by luke and randomized runner to catch failures from other threads:
permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
// needed by randomized runner getTopThreadGroup:
permission java.lang.RuntimePermission "modifyThreadGroup";
// needed by tests e.g. shutting down executors:
permission java.lang.RuntimePermission "modifyThread";
// needed for tons of test hacks etc
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
// test runner seems to require supression of access checks:
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";

// Needed for loading native library (lucene:misc:native) in lucene:misc
// Needed for DirectIODirectory to retrieve block size
permission java.lang.RuntimePermission "getFileStoreAttributes";
permission java.lang.RuntimePermission "writeFileDescriptor";

// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
// Replicator tests connect to ephemeral ports
permission java.net.SocketPermission "127.0.0.1:1024-", "connect,resolve";

// read access to all system properties:
permission java.util.PropertyPermission "*", "read";
Expand All @@ -87,13 +82,6 @@ grant {
permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
};

// Permissions to support ant build
grant {
permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
};

// Permissions for jacoco code coverage
grant {
// permission to write the per-jvm code coverage output
Expand Down Expand Up @@ -122,3 +110,14 @@ grant {
permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
};

// Grant permissions to certain test-related JARs (https://github.com/apache/lucene/pull/13146)
grant codeBase "file:${randomizedtesting.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
permission java.lang.RuntimePermission "modifyThreadGroup";
};
grant codeBase "file:${junit.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
grant codeBase "file:${lucene.test.framework}${/}-" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
84 changes: 21 additions & 63 deletions lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,17 @@
import java.io.OutputStream;
import java.io.PrintStream;
import java.lang.ProcessBuilder.Redirect;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;

/**
Expand All @@ -57,7 +55,8 @@ public void testNRTThreads() throws Exception {
if (System.getProperty("tests.crashmode") == null) {
// try up to 10 times to create an index
for (int i = 0; i < 10; i++) {
forkTest();
final int crashDelayMs = TestUtil.nextInt(random(), 3000, 4000);
forkTest(crashDelayMs);
// if we succeeded in finding an index, we are done.
if (checkIndexes(tempDir)) return;
}
Expand All @@ -68,34 +67,17 @@ public void testNRTThreads() throws Exception {
// assumeFalse("does not support PreFlex, see LUCENE-3992",
// Codec.getDefault().getName().equals("Lucene4x"));

// we are the fork, setup a crashing thread
final int crashTime = TestUtil.nextInt(random(), 3000, 4000);
Thread t =
new Thread() {
@SuppressForbidden(reason = "Thread sleep")
@Override
public void run() {
try {
Thread.sleep(crashTime);
} catch (
@SuppressWarnings("unused")
InterruptedException e) {
}
crashJRE();
}
};
t.setPriority(Thread.MAX_PRIORITY);
t.start();
// run the test until we crash.
for (int i = 0; i < 1000; i++) {
// Effectively an endless loop until the process is killed from the outside.
for (int i = 0; i < Integer.MAX_VALUE; i++) {
super.testNRTThreads();
}
}
}

/** fork ourselves in a new jvm. sets -Dtests.crashmode=true */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
public void forkTest() throws Exception {
@SuppressForbidden(
reason = "ProcessBuilder requires java.io.File for setting ProcessBuilder.directory")
public void forkTest(int crashDelayMs) throws Exception {
List<String> cmd = new ArrayList<>();
cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());
cmd.add("-Xmx512m");
Expand All @@ -114,12 +96,21 @@ public void forkTest() throws Exception {
.redirectInput(Redirect.INHERIT)
.redirectErrorStream(true);
Process p = pb.start();

// We pump everything to stderr.
PrintStream childOut = System.err;
Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
if (VERBOSE) childOut.println(">>> Begin subprocess output");
p.waitFor();
try {
if (VERBOSE) childOut.println(">>> Begin subprocess output");
if (p.waitFor(crashDelayMs, TimeUnit.MILLISECONDS)) {
// This means the process has exited before the timeout. This is odd because it should
// run in an endless loop?
throw new AssertionError("Subprocess has exited unexpectedly.");
}
} finally {
// Try to kill the process and wait until it terminates. destroyForcibly is a no-op in case
// the process is not alive, so no need for additional checks.
p.destroyForcibly().waitFor();
}
stdoutPumper.join();
if (VERBOSE) childOut.println("<<< End subprocess output");
}
Expand All @@ -140,7 +131,7 @@ public void run() {
}
}
} catch (IOException e) {
System.err.println("Couldn't pipe from the forked process: " + e.toString());
System.err.println("Couldn't pipe from the forked process: " + e);
}
}
};
Expand Down Expand Up @@ -186,37 +177,4 @@ public FileVisitResult postVisitDirectory(Path dirPath, IOException exc)
});
return found.get();
}

/** currently, this only works/tested on Sun and IBM. */
@SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
public void crashJRE() {
final String vendor = Constants.JAVA_VENDOR;
final boolean supportsUnsafeNpeDereference =
vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");

try {
if (supportsUnsafeNpeDereference) {
try {
Class<?> clazz = Class.forName("sun.misc.Unsafe");
Field field = clazz.getDeclaredField("theUnsafe");
field.setAccessible(true);
Object o = field.get(null);
Method m = clazz.getMethod("putAddress", long.class, long.class);
m.invoke(o, 0L, 0L);
} catch (Throwable e) {
System.out.println("Couldn't kill the JVM via Unsafe.");
e.printStackTrace(System.out);
}
}

// Fallback attempt to Runtime.halt();
Runtime.getRuntime().halt(-1);
} catch (Exception e) {
System.out.println("Couldn't kill the JVM.");
e.printStackTrace(System.out);
}

// We couldn't get the JVM to crash for some reason.
fail();
}
}
Loading

0 comments on commit 1c77e23

Please sign in to comment.