feat: add fingerprints for JDK classes (#63)

Co-authored-by: Aman Sharma <[email protected]>

terminator-commons/pom.xml

@@ -56,6 +56,11 @@
       <groupId>org.ow2.asm</groupId>
       <artifactId>asm-util</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.github.classgraph</groupId>
+      <artifactId>classgraph</artifactId>
+      <version>4.8.162</version>
+    </dependency>
   </dependencies>
 
   <build>

terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkClass.java

@@ -0,0 +1,8 @@
+package io.github.algomaster99.terminator.commons.fingerprint;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A class that represents a JDK class. It contains the name of the class and the bytes of the class as a {@link ByteBuffer}.
+ */
+public record JdkClass(String name, ByteBuffer bytes) {}

terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/JdkIndexer.java

@@ -0,0 +1,87 @@
+package io.github.algomaster99.terminator.commons.fingerprint;
+
+import io.github.classgraph.ClassGraph;
+import io.github.classgraph.Resource;
+import io.github.classgraph.ScanResult;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import nonapi.io.github.classgraph.classpath.SystemJarFinder;
+
+/**
+ * The JdkIndexer class provides a utility to list all JDK classes by scanning the JDK used for the execution of the application.
+ */
+public class JdkIndexer {
+
+    /**
+     * Returns a list of all JDK classes. The list is populated by scanning the JDK used for the execution of this application.
+     *
+     * @return a list of all JDK classes, never null.
+     */
+    public static List<JdkClass> listJdkClasses() {
+        List<JdkClass> jdkClasses = new ArrayList<>();
+        try (ScanResult scanResult = new ClassGraph()
+                .enableSystemJarsAndModules()
+                .disableDirScanning()
+                .disableJarScanning()
+                .acceptLibOrExtJars()
+                .acceptModules("jdk.*", "java.*")
+                .ignoreClassVisibility()
+                .enableMemoryMapping()
+                .scan(); ) {
+            scanResult.getAllClasses().forEach(classInfo -> {
+                Resource resource = classInfo.getResource();
+                if (resource != null) {
+                    byte[] byteBuffer;
+                    try {
+                        byteBuffer = resource.load();
+                        jdkClasses.add(
+                                new JdkClass(classInfo.getName().replaceAll("\\.", "/"), ByteBuffer.wrap(byteBuffer)));
+                    } catch (IOException e) {
+                        System.err.println("Error loading resource " + resource + ": " + e);
+                    }
+                }
+            });
+        }
+        jdkClasses.addAll(indexJrt());
+        return jdkClasses;
+    }
+
+    /**
+     * Creates an index of the external Jrt jar. This jar provides an API for older jvms to access the modules in the JDK.
+     * The jvm itself does not need this jar.
+     * @return  the list of external jrt jdk classes
+     */
+    private static List<JdkClass> indexJrt() {
+        List<JdkClass> jdkClasses = new ArrayList<>();
+        Set<String> jreLibOrExtJars = SystemJarFinder.getJreLibOrExtJars();
+        for (String path : jreLibOrExtJars) {
+            try {
+                jdkClasses.addAll(readJarFile(path));
+            } catch (Exception e) {
+                System.err.println("Error reading jar file " + path + ": " + e);
+            }
+        }
+        return jdkClasses;
+    }
+
+    private static List<JdkClass> readJarFile(String jarFilePath) throws IOException {
+        List<JdkClass> jdkClasses = new ArrayList<>();
+        try (JarFile jarFile = new JarFile(jarFilePath)) {
+            Enumeration<JarEntry> entries = jarFile.entries();
+            while (entries.hasMoreElements()) {
+                JarEntry entry = entries.nextElement();
+                if (entry.getName().endsWith(".class")) {
+                    byte[] byteBuffer = jarFile.getInputStream(entry).readAllBytes();
+                    jdkClasses.add(new JdkClass(entry.getName().replace(".class", ""), ByteBuffer.wrap(byteBuffer)));
+                }
+            }
+        }
+        return jdkClasses;
+    }
+}

terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/RuntimeClass.java

@@ -20,4 +20,21 @@ public static boolean isSynthetic(byte[] classfileBytes) {
         ClassReader reader = new ClassReader(classfileBytes);
         return (reader.getAccess() & Opcodes.ACC_SYNTHETIC) != 0;
     }
+
+    /**
+     * Skip classes inherited from {@link jdk.internal.reflect.MagicAccessorImpl} because they are generated at runtime using ASM.
+     */
+    public static boolean isGeneratedClassExtendingMagicAccessor(byte[] classfileBytes) {
+        ClassReader reader = new ClassReader(classfileBytes);
+        try {
+            return RuntimeClass.class
+                    .getClassLoader()
+                    .loadClass(reader.getSuperName().replace("/", "."))
+                    .getSuperclass()
+                    .getName()
+                    .equals("jdk.internal.reflect.MagicAccessorImpl");
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
 }

terminator-commons/src/main/java/io/github/algomaster99/terminator/commons/fingerprint/provenance/Jdk.java

@@ -0,0 +1,10 @@
+package io.github.algomaster99.terminator.commons.fingerprint.provenance;
+
+import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassFileAttributes;
+
+public record Jdk(ClassFileAttributes classFileAttributes) implements Provenance {
+    @Override
+    public ClassFileAttributes classFileAttributes() {
+        return classFileAttributes;
+    }
+}

terminator-commons/src/test/java/io/github/algomaster99/terminator/commons/fingerprint/classfile/ClassfileTest.java

@@ -48,6 +48,18 @@ void isSyntheticClass_true() throws IOException {
             assertThat(RuntimeClass.isSynthetic(modifiedBytes)).isTrue();
         }
 
+        @Test
+        void isGeneratedClassExtendingMagicAccessor_true() throws IOException {
+            Path generatedConstructorAccessor = CLASSFILE.resolve("GeneratedConstructorAccessor15.class");
+            assertThat(RuntimeClass.isGeneratedClassExtendingMagicAccessor(
+                            Files.readAllBytes(generatedConstructorAccessor)))
+                    .isTrue();
+
+            Path generatedMethodAccessor = CLASSFILE.resolve("GeneratedMethodAccessor1.class");
+            assertThat(RuntimeClass.isGeneratedClassExtendingMagicAccessor(Files.readAllBytes(generatedMethodAccessor)))
+                    .isTrue();
+        }
+
         private static byte[] makeClassfileSynthetic(byte[] classfileBytes) {
             ClassNode classNode = new ClassNode();
             ClassReader classReader = new ClassReader(classfileBytes);

watchdog-agent/src/main/java/io/github/algomaster99/Options.java

@@ -7,12 +7,20 @@
 import io.github.algomaster99.terminator.commons.cyclonedx.Bom14Schema;
 import io.github.algomaster99.terminator.commons.cyclonedx.Component;
 import io.github.algomaster99.terminator.commons.cyclonedx.CycloneDX;
+import io.github.algomaster99.terminator.commons.fingerprint.JdkIndexer;
+import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassFileAttributes;
+import io.github.algomaster99.terminator.commons.fingerprint.classfile.ClassfileVersion;
+import io.github.algomaster99.terminator.commons.fingerprint.classfile.HashComputer;
+import io.github.algomaster99.terminator.commons.fingerprint.provenance.Jdk;
 import io.github.algomaster99.terminator.commons.fingerprint.provenance.Provenance;
 import io.github.algomaster99.terminator.commons.jar.JarDownloader;
 import java.io.File;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +31,7 @@ public class Options {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(Options.class);
     private Map<String, List<Provenance>> fingerprints = new HashMap<>();
-
+    private Map<String, List<Provenance>> jdkFingerprints = new HashMap<>();
     private boolean skipShutdown = false;
 
     private boolean isSbomPassed = false;
@@ -91,12 +99,17 @@ public Options(String agentArgs) {
         } else {
             LOGGER.info("Taking fingerprint from file: " + fingerprints);
         }
+        processJdk();
     }
 
     public Map<String, List<Provenance>> getFingerprints() {
         return fingerprints;
     }
 
+    public Map<String, List<Provenance>> getJdkFingerprints() {
+        return jdkFingerprints;
+    }
+
     public boolean shouldSkipShutdown() {
         return skipShutdown;
     }
@@ -151,4 +164,50 @@ private void processAllComponents(Bom14Schema sbom) {
             }
         }
     }
+
+    private void processJdk() {
+        JdkIndexer.listJdkClasses().forEach(resource -> {
+            try {
+                byte[] classfileBytes = toArray(resource.bytes());
+                String classfileVersion = ClassfileVersion.getVersion(classfileBytes);
+                String hash = HashComputer.computeHash(classfileBytes, algorithm);
+                jdkFingerprints.computeIfAbsent(
+                        resource.name(),
+                        k -> new ArrayList<>(
+                                List.of((new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm))))));
+                jdkFingerprints.computeIfPresent(resource.name(), (k, v) -> {
+                    v.add(new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm)));
+                    return v;
+                });
+                fingerprints.computeIfAbsent(
+                        resource.name(),
+                        k -> new ArrayList<>(
+                                List.of((new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm))))));
+                fingerprints.computeIfPresent(resource.name(), (k, v) -> {
+                    v.add(new Jdk(new ClassFileAttributes(classfileVersion, hash, algorithm)));
+                    return v;
+                });
+            } catch (NoSuchAlgorithmException e) {
+                LOGGER.error("Failed to compute hash with algorithm: " + algorithm, e);
+                throw new RuntimeException(e);
+            } catch (Exception e) {
+                LOGGER.error("Failed to compute hash for: " + resource, e);
+            }
+        });
+    }
+
+    /**
+     * Converts a bytebuffer to a byte array. If the buffer has an array, it returns it, otherwise it copies the bytes. This is needed because the buffer is not guaranteed to have an array.
+     * See {@link java.nio.ByteBuffer#hasArray()} and {@link java.nio.DirectByteBuffer}.
+     * @param buffer  the buffer to convert
+     * @return  the byte array
+     */
+    private byte[] toArray(ByteBuffer buffer) {
+        if (buffer.hasArray()) {
+            return buffer.array();
+        }
+        byte[] bytes = new byte[buffer.remaining()];
+        buffer.get(bytes);
+        return bytes;
+    }
 }

watchdog-agent/src/main/java/io/github/algomaster99/Terminator.java

@@ -14,9 +14,6 @@
 public class Terminator {
     private static Options options;
 
-    private static final List<String> INTERNAL_PACKAGES =
-            List.of("java/", "javax/", "jdk/", "sun/", "com/sun/", "org/xml/sax", "org/w3c/dom/");
-
     public static void premain(String agentArgs, Instrumentation inst) {
         options = new Options(agentArgs);
         inst.addTransformer(new ClassFileTransformer() {
@@ -27,17 +24,21 @@ public byte[] transform(
                     Class<?> classBeingRedefined,
                     ProtectionDomain protectionDomain,
                     byte[] classfileBuffer) {
+
                 return isLoadedClassWhitelisted(className, classfileBuffer);
             }
         });
     }
 
     private static byte[] isLoadedClassWhitelisted(String className, byte[] classfileBuffer) {
         Map<String, List<Provenance>> fingerprints = options.getFingerprints();
-        if (RuntimeClass.isProxyClass(classfileBuffer)) {
+        if (RuntimeClass.isProxyClass(classfileBuffer)
+                || RuntimeClass.isGeneratedClassExtendingMagicAccessor(classfileBuffer)) {
             return classfileBuffer;
         }
-        if (INTERNAL_PACKAGES.stream().anyMatch(className::startsWith)) {
+        if (className.contains("$")) {
+            // FIXME: we need to check inner classes without loading them. Maybe add the hashes for inner classes in the
+            // fingerprints?
             return classfileBuffer;
         }
         for (String expectedClassName : fingerprints.keySet()) {

watchdog-agent/src/test/java/AgentTest.java

@@ -68,7 +68,8 @@ void shouldDisallowLoadingCustomJDKClass() throws MavenInvocationException, IOEx
     }
 
     @Test
-    void sorald_0_8_5_shouldExitWith_1() throws IOException, InterruptedException {
+    void sorald_0_8_5_shouldExitWith_0() throws IOException, InterruptedException {
+        // contract: sorald 0.8.5 should execute as the SBOM + external jars has every dependency.
         Path project = Paths.get("src/test/resources/sorald-0.8.5");
 
         Path sbom = project.resolve("bom.json");
@@ -93,7 +94,6 @@ void sorald_0_8_5_shouldExitWith_1() throws IOException, InterruptedException {
 
         Process p = pb.start();
         int exitCode = p.waitFor();
-
         assertThat(exitCode).isEqualTo(0);
     }
 
@@ -124,7 +124,6 @@ void spoon_10_4_0_depscan_4_2_2() throws IOException, InterruptedException {
         private int runSpoonWithSbom(Path sbom) throws IOException, InterruptedException {
             Path spoonExecutable = project.resolve("spoon-core-10.4.0-jar-with-dependencies.jar");
             Path workload = project.resolve("Main.java").toAbsolutePath();
-
             String agentArgs = "sbom=" + sbom;
             String[] cmd = {
                 "java",
@@ -206,11 +205,9 @@ private static void deleteContentsOfFile(String file) throws InterruptedExceptio
     private static String getAgentPath(String agentArgs) throws IOException {
         String tempDir = System.getProperty("java.io.tmpdir");
         Path traceCollector = Path.of(tempDir, "watchdog-agent.jar");
-
         try (InputStream traceCollectorStream = Terminator.class.getResourceAsStream("/watchdog-agent.jar")) {
             Files.copy(traceCollectorStream, traceCollector, StandardCopyOption.REPLACE_EXISTING);
         }
-
         return traceCollector.toAbsolutePath() + "=" + agentArgs;
     }
 

watchdog-agent/src/test/java/OptionsTest.java

@@ -1,13 +1,15 @@
 import static io.github.algomaster99.terminator.commons.fingerprint.ParsingHelper.deserializeFingerprints;
 import static org.assertj.core.api.Assertions.assertThat;
 
+import io.github.algomaster99.Options;
 import io.github.algomaster99.terminator.commons.fingerprint.provenance.Jar;
 import io.github.algomaster99.terminator.commons.fingerprint.provenance.Maven;
 import io.github.algomaster99.terminator.commons.fingerprint.provenance.Provenance;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 
@@ -46,4 +48,22 @@ void maven_jar() throws NoSuchMethodException, InvocationTargetException, Illega
                     .hasAtLeastOneElementOfType(Jar.class);
         }
     }
+
+    @Test
+    void verifyIfJdkIndexerFindsJdkClassesDeterministically() throws Exception {
+        // generating 2 times the jdk fingerprint should result in the same fingerprint
+        Options options = new Options("skipShutdown=true");
+        Options options2 = new Options("skipShutdown=true");
+        assertThat(options.getJdkFingerprints()).isNotEmpty();
+        assertThat(options2.getJdkFingerprints()).isNotEmpty();
+        assertThat(options.getJdkFingerprints()).isEqualTo(options2.getJdkFingerprints());
+        System.out.println(options.getJdkFingerprints().size());
+    }
+
+    @Test
+    void verifyJdkIndexerFindsOrgXmlSax() throws Exception {
+        Options options = new Options("skipShutdown=true");
+        var var = options.getJdkFingerprints().keySet().stream().collect(Collectors.toSet());
+        assertThat(var).contains("org/xml/sax/helpers/NamespaceSupport");
+    }
 }