diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle index 4f24d7f412f4..44ae964cab00 100644 --- a/gradle/testing/randomization.gradle +++ b/gradle/testing/randomization.gradle @@ -119,6 +119,18 @@ allprojects { // Resolve test option values after all evaluation is complete. allprojects { plugins.withType(JavaPlugin) { + configurations { + secManagerExclusions + } + dependencies { + secManagerExclusions ( "com.carrotsearch.randomizedtesting:randomizedtesting-runner", { + exclude group: "junit" + }) + secManagerExclusions ( "junit:junit", { + exclude group: "org.hamcrest" + }) + } + afterEvaluate { ext.testOptionsResolved = testOptions.findAll { opt -> propertyOrDefault(opt.propName, opt.value) != null @@ -155,7 +167,7 @@ allprojects { } // Append resolved test properties to the test task. - tasks.withType(Test) { task -> + tasks.withType(Test) { Test task -> // TODO: we could remove some options that are only relevant to the build environment // and not the test JVM itself. systemProperties testOptionsResolved @@ -172,13 +184,13 @@ allprojects { // Enable security manager, if requested. We could move the selection of security manager and security policy // to each project's build/ configuration but it seems compact enough to keep it here for now. - def securityArgumentProvider = new SecurityArgumentProvider(commonDir: project(":lucene").layout.projectDirectory) + def securityArgumentProvider = new SecurityArgumentProvider() + securityArgumentProvider.commonDir = project(":lucene").layout.projectDirectory + securityArgumentProvider.otherProperties = project.objects.mapProperty(String, String) + if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) { if (project.path.endsWith(".tests")) { // LUCENE-10301: for now, do not use the security manager for modular tests (test framework is not available). - } else if (project.path == ":lucene:replicator") { - systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager" - securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/replicator-tests.policy") } else if (project.path.startsWith(":lucene")) { systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager" securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/tests.policy") @@ -189,6 +201,14 @@ allprojects { systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/') systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString() systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString() + + securityArgumentProvider.otherProperties.put("randomizedtesting.jar", project.provider { -> + return configurations.secManagerExclusions.resolve().find { it.name.startsWith("randomizedtesting-runner-") }.absolutePath + }) + securityArgumentProvider.otherProperties.put("junit.jar", project.provider { -> + return configurations.secManagerExclusions.resolve().find { it.name.startsWith("junit-") }.absolutePath + }) + securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath) } doFirst { @@ -242,12 +262,20 @@ class SecurityArgumentProvider implements CommandLineArgumentProvider { @PathSensitive(PathSensitivity.RELATIVE) RegularFile javaSecurityPolicy + @Input + MapProperty otherProperties + @Override Iterable asArguments() { def args = ["-Dcommon.dir=${commonDir.getAsFile()}"] if (javaSecurityPolicy) { args.add("-Djava.security.policy=${javaSecurityPolicy.getAsFile()}") } + + otherProperties.getOrElse(Map.of()).forEach { key, value -> + args.add("-D${key}=${value}") + } + return args } } diff --git a/gradle/testing/randomization/policies/replicator-tests.policy b/gradle/testing/randomization/policies/replicator-tests.policy deleted file mode 100644 index 4d8fcd83f242..000000000000 --- a/gradle/testing/randomization/policies/replicator-tests.policy +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Policy file for :lucene:replicator tests. Please keep minimal and avoid wildcards. -// this differs from the standard lucene policy in that it must allow read-write access -// to all system properties, because of jetty calls to System.getProperties() - -grant { - // jetty-specific: - permission java.lang.RuntimePermission "getenv.JETTY_AVAILABLE_PROCESSORS"; - permission java.lang.RuntimePermission "getenv.JETTY_WORKER_INSTANCE"; - // allow TestNRTReplication fork its jvm - permission java.io.FilePermission "${java.home}${/}-", "read,execute"; - // read/write access to all system properties (required by jetty in these tests) - permission java.util.PropertyPermission "*", "read,write"; - // all possibilities of accepting/binding/connecting on localhost with ports >= 1024: - permission java.net.SocketPermission "localhost:1024-", "accept,connect,resolve"; - permission java.net.SocketPermission "127.0.0.1:1024-", "accept,connect,resolve"; - permission java.net.SocketPermission "[::1]:1024-", "accept,connect,resolve"; - - // test-files/ resources - permission java.io.FilePermission "${common.dir}${/}-", "read"; - - // write only to sandbox - permission java.io.FilePermission "${tests.linedocsfile}", "read"; - permission java.io.FilePermission "${java.io.tmpdir}", "read,write"; - permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete"; - - // needed by randomizedtesting runner to identify test methods. - permission java.lang.RuntimePermission "accessDeclaredMembers"; - // needed by certain tests to redirect sysout/syserr: - permission java.lang.RuntimePermission "setIO"; - // needed by randomized runner to catch failures from other threads: - permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler"; - // needed by randomized runner getTopThreadGroup: - permission java.lang.RuntimePermission "modifyThreadGroup"; - // needed by tests e.g. shutting down executors: - permission java.lang.RuntimePermission "modifyThread"; - // needed for tons of test hacks etc - permission java.lang.RuntimePermission "getStackTrace"; - // needed for mock filesystems in tests - permission java.lang.RuntimePermission "fileSystemProvider"; - // test runner seems to require supression of access checks: - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; -}; - -// Permissions to support ant build -grant { - permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read"; - permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete"; - permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete"; -}; - -// Permissions for jacoco code coverage -grant { - // permission to write the per-jvm code coverage output - permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write"; - // needed by jacoco to dump coverage on shutdown - permission java.lang.RuntimePermission "shutdownHooks"; - // needed by jacoco to instrument classes - permission java.lang.RuntimePermission "defineClass"; - // needed by jacoco for God knows what. - permission java.lang.RuntimePermission "createClassLoader"; -}; - -// Grant all permissions to Gradle test runner classes. -grant codeBase "file:${gradle.lib.dir}${/}-" { - permission java.security.AllPermission; -}; - -grant codeBase "file:${gradle.worker.jar}" { - permission java.security.AllPermission; -}; - -grant { - // Allow reading gradle worker JAR. - permission java.io.FilePermission "${gradle.worker.jar}", "read"; - // Allow reading from classpath JARs (resources). - permission java.io.FilePermission "${gradle.user.home}${/}-", "read"; -}; diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy index ffb6d0b52d0f..f8e09ba03661 100644 --- a/gradle/testing/randomization/policies/tests.policy +++ b/gradle/testing/randomization/policies/tests.policy @@ -33,34 +33,29 @@ grant { // misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created permission java.nio.file.LinkPermission "hard"; - // needed by SSD detection tests in TestIOUtils (creates symlinks) - permission java.nio.file.LinkPermission "symbolic"; - // needed by randomizedtesting runner to identify test methods. + // needed by RamUsageEstimator, and randomizedtesting runner to identify test methods. permission java.lang.RuntimePermission "accessDeclaredMembers"; // needed by certain tests to capture sysout/syserr: permission java.lang.RuntimePermission "setIO"; - // needed by randomized runner to catch failures from other threads: + // needed by luke and randomized runner to catch failures from other threads: permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler"; - // needed by randomized runner getTopThreadGroup: - permission java.lang.RuntimePermission "modifyThreadGroup"; // needed by tests e.g. shutting down executors: permission java.lang.RuntimePermission "modifyThread"; // needed for tons of test hacks etc permission java.lang.RuntimePermission "getStackTrace"; // needed for mock filesystems in tests permission java.lang.RuntimePermission "fileSystemProvider"; - // test runner seems to require supression of access checks: - permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; // needed by cyberneko usage by benchmarks on J9 permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util"; - // Needed for loading native library (lucene:misc:native) in lucene:misc + // Needed for DirectIODirectory to retrieve block size permission java.lang.RuntimePermission "getFileStoreAttributes"; - permission java.lang.RuntimePermission "writeFileDescriptor"; // TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range): permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve"; + // Replicator tests connect to ephemeral ports + permission java.net.SocketPermission "127.0.0.1:1024-", "connect,resolve"; // read access to all system properties: permission java.util.PropertyPermission "*", "read"; @@ -87,13 +82,6 @@ grant { permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read"; }; -// Permissions to support ant build -grant { - permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read"; - permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete"; - permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete"; -}; - // Permissions for jacoco code coverage grant { // permission to write the per-jvm code coverage output @@ -122,3 +110,14 @@ grant { permission java.io.FilePermission "${gradle.user.home}${/}-", "read"; }; +// Grant permissions to certain test-related JARs (https://github.com/apache/lucene/pull/13146) +grant codeBase "file:${randomizedtesting.jar}" { + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.RuntimePermission "modifyThreadGroup"; +}; +grant codeBase "file:${junit.jar}" { + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; +grant codeBase "file:${lucene.test.framework}${/}-" { + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; +}; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java index 296a39af15d2..60de20d22bbd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java @@ -22,8 +22,6 @@ import java.io.OutputStream; import java.io.PrintStream; import java.lang.ProcessBuilder.Redirect; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -31,10 +29,10 @@ import java.nio.file.SimpleFileVisitor; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.tests.store.BaseDirectoryWrapper; import org.apache.lucene.tests.util.TestUtil; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.SuppressForbidden; /** @@ -57,7 +55,8 @@ public void testNRTThreads() throws Exception { if (System.getProperty("tests.crashmode") == null) { // try up to 10 times to create an index for (int i = 0; i < 10; i++) { - forkTest(); + final int crashDelayMs = TestUtil.nextInt(random(), 3000, 4000); + forkTest(crashDelayMs); // if we succeeded in finding an index, we are done. if (checkIndexes(tempDir)) return; } @@ -68,34 +67,17 @@ public void testNRTThreads() throws Exception { // assumeFalse("does not support PreFlex, see LUCENE-3992", // Codec.getDefault().getName().equals("Lucene4x")); - // we are the fork, setup a crashing thread - final int crashTime = TestUtil.nextInt(random(), 3000, 4000); - Thread t = - new Thread() { - @SuppressForbidden(reason = "Thread sleep") - @Override - public void run() { - try { - Thread.sleep(crashTime); - } catch ( - @SuppressWarnings("unused") - InterruptedException e) { - } - crashJRE(); - } - }; - t.setPriority(Thread.MAX_PRIORITY); - t.start(); - // run the test until we crash. - for (int i = 0; i < 1000; i++) { + // Effectively an endless loop until the process is killed from the outside. + for (int i = 0; i < Integer.MAX_VALUE; i++) { super.testNRTThreads(); } } } /** fork ourselves in a new jvm. sets -Dtests.crashmode=true */ - @SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD") - public void forkTest() throws Exception { + @SuppressForbidden( + reason = "ProcessBuilder requires java.io.File for setting ProcessBuilder.directory") + public void forkTest(int crashDelayMs) throws Exception { List cmd = new ArrayList<>(); cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString()); cmd.add("-Xmx512m"); @@ -114,12 +96,21 @@ public void forkTest() throws Exception { .redirectInput(Redirect.INHERIT) .redirectErrorStream(true); Process p = pb.start(); - // We pump everything to stderr. PrintStream childOut = System.err; Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut); - if (VERBOSE) childOut.println(">>> Begin subprocess output"); - p.waitFor(); + try { + if (VERBOSE) childOut.println(">>> Begin subprocess output"); + if (p.waitFor(crashDelayMs, TimeUnit.MILLISECONDS)) { + // This means the process has exited before the timeout. This is odd because it should + // run in an endless loop? + throw new AssertionError("Subprocess has exited unexpectedly."); + } + } finally { + // Try to kill the process and wait until it terminates. destroyForcibly is a no-op in case + // the process is not alive, so no need for additional checks. + p.destroyForcibly().waitFor(); + } stdoutPumper.join(); if (VERBOSE) childOut.println("<<< End subprocess output"); } @@ -140,7 +131,7 @@ public void run() { } } } catch (IOException e) { - System.err.println("Couldn't pipe from the forked process: " + e.toString()); + System.err.println("Couldn't pipe from the forked process: " + e); } } }; @@ -186,37 +177,4 @@ public FileVisitResult postVisitDirectory(Path dirPath, IOException exc) }); return found.get(); } - - /** currently, this only works/tested on Sun and IBM. */ - @SuppressForbidden(reason = "We need Unsafe to actually crush :-)") - public void crashJRE() { - final String vendor = Constants.JAVA_VENDOR; - final boolean supportsUnsafeNpeDereference = - vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple"); - - try { - if (supportsUnsafeNpeDereference) { - try { - Class clazz = Class.forName("sun.misc.Unsafe"); - Field field = clazz.getDeclaredField("theUnsafe"); - field.setAccessible(true); - Object o = field.get(null); - Method m = clazz.getMethod("putAddress", long.class, long.class); - m.invoke(o, 0L, 0L); - } catch (Throwable e) { - System.out.println("Couldn't kill the JVM via Unsafe."); - e.printStackTrace(System.out); - } - } - - // Fallback attempt to Runtime.halt(); - Runtime.getRuntime().halt(-1); - } catch (Exception e) { - System.out.println("Couldn't kill the JVM."); - e.printStackTrace(System.out); - } - - // We couldn't get the JVM to crash for some reason. - fail(); - } } diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java index c715bd579362..f21ac11b8846 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java @@ -21,7 +21,6 @@ import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Method; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -45,7 +44,6 @@ import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites; import org.apache.lucene.tests.util.TestUtil; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.SuppressForbidden; import org.junit.Assume; @@ -58,7 +56,6 @@ */ @SuppressCodecs({"MockRandom", "Direct", "SimpleText"}) @SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure") -@SuppressForbidden(reason = "We need Unsafe to actually crush :-)") public class TestSimpleServer extends LuceneTestCase { static final Set clientThreads = Collections.synchronizedSet(new HashSet<>()); @@ -135,39 +132,12 @@ public void run() { } } - /** currently, this only works/tested on Sun and IBM. */ + /** forcibly halt the JVM: similar to crashing */ // poached from TestIndexWriterOnJRECrash ... should we factor out to TestUtil? seems dangerous // to give it such "publicity"? private static void crashJRE() { - final String vendor = Constants.JAVA_VENDOR; - final boolean supportsUnsafeNpeDereference = - vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple"); - - try { - if (supportsUnsafeNpeDereference) { - try { - Class clazz = Class.forName("sun.misc.Unsafe"); - java.lang.reflect.Field field = clazz.getDeclaredField("theUnsafe"); - field.setAccessible(true); - Object o = field.get(null); - Method m = clazz.getMethod("putAddress", long.class, long.class); - m.invoke(o, 0L, 0L); - } catch (Throwable e) { - System.out.println("Couldn't kill the JVM via Unsafe."); - e.printStackTrace(System.out); - } - } - - // Fallback attempt to Runtime.halt(); - Runtime.getRuntime().halt(-1); - } catch (Exception e) { - System.out.println("Couldn't kill the JVM."); - e.printStackTrace(System.out); - } - - // We couldn't get the JVM to crash for some reason. - throw new RuntimeException("JVM refuses to die!"); + Runtime.getRuntime().halt(1); } static void writeFilesMetaData(DataOutput out, Map files)