diff --git a/.gitignore b/.gitignore index 4ccfd6f..c9a88df 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ cql.txt env/data env/import .DS_Store +jre_libs diff --git a/build.gradle b/build.gradle index 707b4f9..a541cb5 100644 --- a/build.gradle +++ b/build.gradle @@ -1,12 +1,15 @@ plugins { id 'java' - id 'org.springframework.boot' version '2.7.7' - id 'io.spring.dependency-management' version '1.1.0' + id 'org.springframework.boot' version '3.2.1' + id 'io.spring.dependency-management' version '1.1.4' } group = 'tabby' -version = '1.2.0.RELEASE' -sourceCompatibility = '1.8' +version = '1.3.0' + +java { + sourceCompatibility = '17' +} configurations { compileOnly { @@ -31,9 +34,11 @@ dependencies { implementation 'org.soot-oss:soot:4.4.1' // implementation 'ca.mcgill.sable:soot:4.0.0' - implementation 'com.google.code.gson:gson:2.8.9' + implementation 'com.google.code.gson:gson:2.10.1' implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.tomcat:tomcat-jasper:9.0.56' + + // jsp plugin dependencies + implementation 'org.apache.tomcat:tomcat-jasper:10.1.18' implementation 'org.apache.ant:ant:1.10.12' compileOnly 'org.slf4j:slf4j-api' diff --git a/config/settings.properties b/config/settings.properties index ff4c3e4..c7b9d92 100644 --- a/config/settings.properties +++ b/config/settings.properties @@ -8,9 +8,10 @@ tabby.output.directory = ./output/dev tabby.debug.details = false # jdk settings +tabby.build.isJRE9Module = false +tabby.build.javaHome = /Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home tabby.build.isJDKProcess = false tabby.build.withAllJDK = false -tabby.build.excludeJDK = false tabby.build.isJDKOnly = false # dealing fatjar diff --git "a/doc/Tabby \351\205\215\347\275\256\346\226\207\344\273\266\344\273\213\347\273\215.md" "b/doc/Tabby \351\205\215\347\275\256\346\226\207\344\273\266\344\273\213\347\273\215.md" index cd404d6..527b7a6 100644 --- "a/doc/Tabby \351\205\215\347\275\256\346\226\207\344\273\266\344\273\213\347\273\215.md" +++ "b/doc/Tabby \351\205\215\347\275\256\346\226\207\344\273\266\344\273\213\347\273\215.md" @@ -33,11 +33,13 @@ tabby.build.isNeedToCreateIgnoreList = false #### 配置 jdk 依赖是否参与分析 ```properties # jdk settings +tabby.build.isJRE9Module = false # 指代下述javaHome版本是否 >= 9 +tabby.build.javaHome = /Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home tabby.build.isJDKProcess = false # 分析过程是否加入基础的2个jdk依赖 tabby.build.withAllJDK = false # 分析过程是否加入全量的jdk依赖 -tabby.build.excludeJDK = false # 分析过程是否剔除当前运行环境的jdk依赖，此时target目录需要提供需要分析的jdk依赖 tabby.build.isJDKOnly = false # 分析过程是否仅分析jdk依赖，不会去分析target目录下的文件 ``` +如果 javaHome 所指向的路径是 jdk >= 9, 第一次生成jre_libs目录会花一点时间，请耐心等待。 #### 配置分析目标 diff --git a/env/tabby-path-finder-1.0.jar b/env/tabby-path-finder-1.0.jar deleted file mode 100644 index 9a55ebb..0000000 Binary files a/env/tabby-path-finder-1.0.jar and /dev/null differ diff --git a/env/tabby-path-finder-1.1.jar b/env/tabby-path-finder-1.1.jar new file mode 100644 index 0000000..568a585 Binary files /dev/null and b/env/tabby-path-finder-1.1.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f371643..070cb70 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/run.sh b/run.sh index 262e11d..9b3b225 100755 --- a/run.sh +++ b/run.sh @@ -1,6 +1,5 @@ if [ $1 = 'build' ] then - echo "start to run tabby" java -Xmx16g -jar build/libs/tabby.jar elif [ $1 = 'load' ] then diff --git a/src/main/java/tabby/App.java b/src/main/java/tabby/App.java index 0d9fd27..d746c97 100644 --- a/src/main/java/tabby/App.java +++ b/src/main/java/tabby/App.java @@ -10,9 +10,11 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import sun.misc.Signal; +import tabby.common.utils.FileUtils; import tabby.config.GlobalConfiguration; import tabby.core.Analyser; -import tabby.common.utils.FileUtils; @Slf4j @SpringBootApplication @@ -30,6 +32,21 @@ public static void main(String[] args) { SpringApplication.run(App.class, args).close(); } + public void setJavaHome(){ + // set java home + if(GlobalConfiguration.TARGET_JAVA_HOME == null){ + String javaHome = System.getProperty("java.home"); + if(javaHome == null){ + javaHome = System.getenv("JAVA_HOME"); + } + if(javaHome != null){ + GlobalConfiguration.TARGET_JAVA_HOME = javaHome; + } + } + + log.info("Analysis target java.home: " + GlobalConfiguration.TARGET_JAVA_HOME); + } + public void setLogDebugLevel(){ LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); if(GlobalConfiguration.DEBUG) { @@ -42,7 +59,14 @@ CommandLineRunner run(){ return args -> { try{ GlobalConfiguration.initConfig(); + setJavaHome(); setLogDebugLevel(); + Signal.handle(new Signal("INT"), // SIGINT + signal -> { + log.error("Force Stop by control+c"); + stopThreads(GlobalConfiguration.tabbyCollectorExecutor); + System.exit(0); + }); analyser.run(); }catch (IllegalArgumentException e){ log.error(e.getMessage() + ", Please check your settings.properties file."); @@ -50,4 +74,10 @@ CommandLineRunner run(){ log.info("Done. Bye!"); }; } + + public static void stopThreads(ThreadPoolTaskExecutor executor){ + if(executor != null && executor.getActiveCount() > 0){ + executor.shutdown(); + } + } } diff --git a/src/main/java/tabby/analysis/data/FileLocation.java b/src/main/java/tabby/analysis/data/FileLocation.java index e39c3f5..f34e649 100644 --- a/src/main/java/tabby/analysis/data/FileLocation.java +++ b/src/main/java/tabby/analysis/data/FileLocation.java @@ -4,7 +4,7 @@ import org.apache.commons.lang3.RandomStringUtils; import tabby.config.GlobalConfiguration; import tabby.common.utils.FileUtils; -import tabby.plugin.JspCompilePlugin; +import tabby.plugin.jsp.JspCompilePlugin; import java.io.File; import java.io.IOException; diff --git a/src/main/java/tabby/common/bean/converter/List2JsonStringConverter.java b/src/main/java/tabby/common/bean/converter/List2JsonStringConverter.java index a0d2553..e8b79aa 100644 --- a/src/main/java/tabby/common/bean/converter/List2JsonStringConverter.java +++ b/src/main/java/tabby/common/bean/converter/List2JsonStringConverter.java @@ -3,8 +3,8 @@ import com.google.gson.reflect.TypeToken; import tabby.config.GlobalConfiguration; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/tabby/common/bean/converter/ListInteger2JsonStringConverter.java b/src/main/java/tabby/common/bean/converter/ListInteger2JsonStringConverter.java index d77881d..9cf0a9d 100644 --- a/src/main/java/tabby/common/bean/converter/ListInteger2JsonStringConverter.java +++ b/src/main/java/tabby/common/bean/converter/ListInteger2JsonStringConverter.java @@ -3,8 +3,8 @@ import com.google.gson.reflect.TypeToken; import tabby.config.GlobalConfiguration; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/tabby/common/bean/converter/ListSet2JsonStringConverter.java b/src/main/java/tabby/common/bean/converter/ListSet2JsonStringConverter.java index 6814781..bb0a9c6 100644 --- a/src/main/java/tabby/common/bean/converter/ListSet2JsonStringConverter.java +++ b/src/main/java/tabby/common/bean/converter/ListSet2JsonStringConverter.java @@ -3,8 +3,8 @@ import com.google.gson.reflect.TypeToken; import tabby.config.GlobalConfiguration; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/tabby/common/bean/converter/Map2JsonStringConverter.java b/src/main/java/tabby/common/bean/converter/Map2JsonStringConverter.java index cee14cf..056e3d8 100644 --- a/src/main/java/tabby/common/bean/converter/Map2JsonStringConverter.java +++ b/src/main/java/tabby/common/bean/converter/Map2JsonStringConverter.java @@ -3,8 +3,8 @@ import com.google.gson.reflect.TypeToken; import tabby.config.GlobalConfiguration; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/tabby/common/bean/converter/Map2JsonStringForAnnotationsConverter.java b/src/main/java/tabby/common/bean/converter/Map2JsonStringForAnnotationsConverter.java index ca3e9ed..a0a9bd5 100644 --- a/src/main/java/tabby/common/bean/converter/Map2JsonStringForAnnotationsConverter.java +++ b/src/main/java/tabby/common/bean/converter/Map2JsonStringForAnnotationsConverter.java @@ -3,8 +3,8 @@ import com.google.gson.reflect.TypeToken; import tabby.config.GlobalConfiguration; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/tabby/common/bean/converter/MethodRef2StringConverter.java b/src/main/java/tabby/common/bean/converter/MethodRef2StringConverter.java index 3422889..0fefb6d 100644 --- a/src/main/java/tabby/common/bean/converter/MethodRef2StringConverter.java +++ b/src/main/java/tabby/common/bean/converter/MethodRef2StringConverter.java @@ -2,8 +2,8 @@ import tabby.common.bean.ref.MethodReference; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; /** * @author wh1t3P1g diff --git a/src/main/java/tabby/common/bean/edge/Alias.java b/src/main/java/tabby/common/bean/edge/Alias.java index d2da664..c60ebef 100644 --- a/src/main/java/tabby/common/bean/edge/Alias.java +++ b/src/main/java/tabby/common/bean/edge/Alias.java @@ -6,10 +6,10 @@ import tabby.common.bean.ref.MethodReference; import tabby.common.utils.HashingUtils; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.Table; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.Table; /** * @author wh1t3P1g diff --git a/src/main/java/tabby/common/bean/edge/Call.java b/src/main/java/tabby/common/bean/edge/Call.java index 3e9333f..9a2076d 100644 --- a/src/main/java/tabby/common/bean/edge/Call.java +++ b/src/main/java/tabby/common/bean/edge/Call.java @@ -8,7 +8,7 @@ import tabby.common.bean.ref.MethodReference; import tabby.common.utils.HashingUtils; -import javax.persistence.*; +import jakarta.persistence.*; import java.util.*; /** diff --git a/src/main/java/tabby/common/bean/edge/Extend.java b/src/main/java/tabby/common/bean/edge/Extend.java index e837d8e..7588eb7 100644 --- a/src/main/java/tabby/common/bean/edge/Extend.java +++ b/src/main/java/tabby/common/bean/edge/Extend.java @@ -6,7 +6,7 @@ import tabby.common.bean.ref.ClassReference; import tabby.common.utils.HashingUtils; -import javax.persistence.*; +import jakarta.persistence.*; /** * @author wh1t3P1g diff --git a/src/main/java/tabby/common/bean/edge/Has.java b/src/main/java/tabby/common/bean/edge/Has.java index 1785d94..356c405 100644 --- a/src/main/java/tabby/common/bean/edge/Has.java +++ b/src/main/java/tabby/common/bean/edge/Has.java @@ -8,7 +8,7 @@ import tabby.common.bean.ref.MethodReference; import tabby.common.utils.HashingUtils; -import javax.persistence.*; +import jakarta.persistence.*; /** * @author wh1t3P1g diff --git a/src/main/java/tabby/common/bean/edge/Interfaces.java b/src/main/java/tabby/common/bean/edge/Interfaces.java index 76193f3..5fde78a 100644 --- a/src/main/java/tabby/common/bean/edge/Interfaces.java +++ b/src/main/java/tabby/common/bean/edge/Interfaces.java @@ -6,10 +6,10 @@ import tabby.common.bean.ref.ClassReference; import tabby.common.utils.HashingUtils; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.Table; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.Table; /** * @author wh1t3P1g diff --git a/src/main/java/tabby/common/bean/ref/ClassReference.java b/src/main/java/tabby/common/bean/ref/ClassReference.java index 5986e0a..d81d3fd 100644 --- a/src/main/java/tabby/common/bean/ref/ClassReference.java +++ b/src/main/java/tabby/common/bean/ref/ClassReference.java @@ -10,7 +10,7 @@ import tabby.common.bean.converter.List2JsonStringConverter; import tabby.common.bean.converter.Map2JsonStringForAnnotationsConverter; -import javax.persistence.*; +import jakarta.persistence.*; import java.nio.charset.StandardCharsets; import java.util.*; diff --git a/src/main/java/tabby/common/bean/ref/MethodReference.java b/src/main/java/tabby/common/bean/ref/MethodReference.java index ea719d4..de48816 100644 --- a/src/main/java/tabby/common/bean/ref/MethodReference.java +++ b/src/main/java/tabby/common/bean/ref/MethodReference.java @@ -13,7 +13,7 @@ import tabby.common.bean.edge.Call; import tabby.common.utils.SemanticUtils; -import javax.persistence.*; +import jakarta.persistence.*; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/main/java/tabby/common/utils/FileUtils.java b/src/main/java/tabby/common/utils/FileUtils.java index c550e5e..8df471e 100644 --- a/src/main/java/tabby/common/utils/FileUtils.java +++ b/src/main/java/tabby/common/utils/FileUtils.java @@ -4,11 +4,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import tabby.config.GlobalConfiguration; +import tabby.plugin.jmod.JModTransferPlugin; import java.io.*; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.jar.JarFile; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -299,4 +302,109 @@ public static String getRealPath(String filepath) throws IllegalArgumentExceptio throw new IllegalArgumentException("Cache Path error!"); } } + + public static void copy(String source, String target) throws IOException { + Files.copy(Paths.get(source), Paths.get(target), StandardCopyOption.REPLACE_EXISTING); + } + + public static Set findAllJdkDependencies(JModTransferPlugin plugin){ + String javaHome = GlobalConfiguration.TARGET_JAVA_HOME; + + if(javaHome == null){ + throw new RuntimeException("JAVA_HOME not set!"); + } + + Set targets = new HashSet<>(); + if(GlobalConfiguration.IS_JRE9_MODULE){ + targets.add(String.join(File.separator, Arrays.asList(javaHome, "jmods"))); + }else{ + targets.add(String.join(File.separator, Arrays.asList(javaHome, "lib"))); + targets.add(String.join(File.separator, Arrays.asList(javaHome, "jre", "lib"))); + } + + Set> futures = new HashSet<>(); + Set libraries = new HashSet<>(); + for(String target:targets){ + try{ + Path path = Paths.get(target).toRealPath(); + String realPath = path.toString(); + Files.walkFileTree(path, new SimpleFileVisitor(){ + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + String source = path.toAbsolutePath().toString(); + String filename = path.getFileName().toString(); + if(source.endsWith("local_policy.jar") || source.endsWith("US_export_policy.jar")){ + return FileVisitResult.CONTINUE; + } + + if(source.endsWith(".jar")){ + String dest = String.join(File.separator, Arrays.asList(GlobalConfiguration.JRE_LIBS_PATH, filename)); + futures.add(plugin.transfer(source, dest)); + libraries.add(dest); + }else if(source.endsWith(".jmod")){ + String dest = String.join(File.separator, Arrays.asList(GlobalConfiguration.JRE_LIBS_PATH, filename+".jar")); + futures.add(plugin.transfer(source, dest)); + libraries.add(dest); + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if(!GlobalConfiguration.IS_JRE9_MODULE || realPath.equals(dir.toAbsolutePath().toString())){ + return super.preVisitDirectory(dir, attrs); + }else{ + return FileVisitResult.SKIP_SUBTREE; + } + } + }); + }catch (Exception e){ + e.printStackTrace(); + } + } + + // wait for finished + for(CompletableFuture future:futures){ + try { + future.get(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + return libraries; + } + + public static Set findAllJarFiles(String target, boolean isNeedRecursion) throws IOException { + Set paths = new HashSet<>(); + Path path = Paths.get(target).toRealPath(); + String realPath = path.toString(); + if (!Files.exists(path)) { + throw new IllegalArgumentException("Invalid target path: " + path); + } + + Files.walkFileTree(path, new SimpleFileVisitor(){ + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + String file = path.toAbsolutePath().toString(); + if(file.endsWith(".jar") || file.endsWith(".jmod")){ + paths.add(file); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if(isNeedRecursion || realPath.equals(dir.toAbsolutePath().toString())){ + return super.preVisitDirectory(dir, attrs); + }else{ + return FileVisitResult.SKIP_SUBTREE; + } + } + }); + + return paths; + } } diff --git a/src/main/java/tabby/config/AsyncConfiguration.java b/src/main/java/tabby/config/AsyncConfiguration.java index de40e0f..74c8312 100644 --- a/src/main/java/tabby/config/AsyncConfiguration.java +++ b/src/main/java/tabby/config/AsyncConfiguration.java @@ -9,9 +9,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; -import static java.lang.Math.max; -import static java.lang.Math.min; - /** * @author wh1t3P1g * @since 2021/4/23 @@ -27,16 +24,19 @@ public class AsyncConfiguration { public Executor master() { int corePoolSize = (int) (CORE_POOL_SIZE / 0.8); int maxPoolSize = (int) (CORE_POOL_SIZE / 0.6); - return makeExecutor(corePoolSize, maxPoolSize,"tabby-collector"); + ThreadPoolTaskExecutor executor = makeExecutor(corePoolSize, maxPoolSize,"tabby-collector"); + GlobalConfiguration.tabbyCollectorExecutor = executor; + return executor; } - private Executor makeExecutor(int corePoolSize, int maxPoolSize, String prefix){ + private ThreadPoolTaskExecutor makeExecutor(int corePoolSize, int maxPoolSize, String prefix){ log.info("Open {} size for thread pool {}", corePoolSize, prefix); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(corePoolSize); executor.setMaxPoolSize(maxPoolSize); executor.setQueueCapacity(maxPoolSize * 1000); executor.setKeepAliveSeconds(300); + executor.setAwaitTerminationSeconds(1); executor.setThreadNamePrefix(prefix+"-"); executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); executor.initialize(); diff --git a/src/main/java/tabby/config/GlobalConfiguration.java b/src/main/java/tabby/config/GlobalConfiguration.java index 8b3c826..c9850e9 100644 --- a/src/main/java/tabby/config/GlobalConfiguration.java +++ b/src/main/java/tabby/config/GlobalConfiguration.java @@ -3,6 +3,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import tabby.core.container.RulesContainer; import tabby.common.utils.FileUtils; @@ -24,6 +25,7 @@ public class GlobalConfiguration { public static String CONFIG_FILE_PATH = String.join(File.separator, System.getProperty("user.dir"), "config", "settings.properties"); public static String LIBS_PATH = String.join(File.separator, System.getProperty("user.dir"), "libs"); + public static String JRE_LIBS_PATH = String.join(File.separator, System.getProperty("user.dir"), "jre_libs"); public static String RULES_PATH; public static String SINK_RULE_PATH; public static String SYSTEM_RULE_PATH; @@ -51,7 +53,6 @@ public class GlobalConfiguration { public static boolean IS_WEB_MODE = false; public static boolean IS_JDK_ONLY = false; public static boolean IS_JDK_PROCESS = false; - public static boolean IS_EXCLUDE_JDK = false; public static boolean IS_WITH_ALL_JDK = false; public static boolean IS_CHECK_FAT_JAR = false; public static boolean IS_FULL_CALL_GRAPH_CONSTRUCT = false; @@ -59,8 +60,16 @@ public class GlobalConfiguration { private static Properties props; public static boolean isInitialed = false; public static boolean isNeedStop = false; - + public static boolean IS_JRE9_MODULE = false; + public static String TARGET_JAVA_HOME = null; public static String THREAD_POOL_SIZE = "max"; + public static ThreadPoolTaskExecutor tabbyCollectorExecutor; + + static { + if(!FileUtils.fileExists(JRE_LIBS_PATH)){ + FileUtils.createDirectory(JRE_LIBS_PATH); + } + } public static void init(){ if(props == null){ @@ -81,7 +90,8 @@ public static void init(){ BASIC_CLASSES_PATH = String.join(File.separator, RULES_PATH, "basicClasses.json"); COMMON_JARS_PATH = String.join(File.separator, RULES_PATH, "commonJars.json"); THREAD_POOL_SIZE = getProperty("tabby.build.thread.size", "max", props); - + IS_JRE9_MODULE = getBooleanProperty("tabby.build.isJRE9Module", "false", props); + TARGET_JAVA_HOME = getProperty("tabby.build.javaHome", null, props); int maxThreadPoolSize = Runtime.getRuntime().availableProcessors(); if("max".equals(THREAD_POOL_SIZE)){ AsyncConfiguration.CORE_POOL_SIZE = maxThreadPoolSize; @@ -131,11 +141,9 @@ public static void initConfig(){ if(IS_JDK_ONLY){ IS_WITH_ALL_JDK = true; - IS_EXCLUDE_JDK = false; IS_JDK_PROCESS = true; }else{ IS_WITH_ALL_JDK = getBooleanProperty("tabby.build.withAllJDK", "false", props); - IS_EXCLUDE_JDK = getBooleanProperty("tabby.build.excludeJDK", "false", props); IS_JDK_PROCESS = getBooleanProperty("tabby.build.isJDKProcess", "false", props); } @@ -189,7 +197,11 @@ public static void clean(String directory){ } public static String getProperty(String key, String defaultValue, Properties props){ - return props.getProperty(key, defaultValue).trim(); + String data = props.getProperty(key, defaultValue); + if(data != null){ + return data.trim(); + } + return null; } public static boolean getBooleanProperty(String key, String defaultValue, Properties props){ diff --git a/src/main/java/tabby/config/SootConfiguration.java b/src/main/java/tabby/config/SootConfiguration.java index 6e272f4..23b5640 100644 --- a/src/main/java/tabby/config/SootConfiguration.java +++ b/src/main/java/tabby/config/SootConfiguration.java @@ -27,7 +27,7 @@ public static void initSootOption(){ Options.v().set_src_prec(Options.src_prec_class); // 优先处理class格式 // Options.v().set_output_dir(output); // 设置IR Jimple的输出目录 // Options.v().set_output_format(Options.output_format_jimple); // 输出Jimple格式 -// Options.v().set_on_the_fly(true); + Options.v().set_on_the_fly(true); // Options.v().set_no_output_source_file_attribute(true); // Options.v().set_validate(true); // Options.v().set_ignore_classpath_errors(true); // Ignores invalid entries on the Soot classpath. diff --git a/src/main/java/tabby/core/Analyser.java b/src/main/java/tabby/core/Analyser.java index 4f70bd3..72d7da4 100644 --- a/src/main/java/tabby/core/Analyser.java +++ b/src/main/java/tabby/core/Analyser.java @@ -4,10 +4,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import soot.CompilationDeathException; -import soot.G; import soot.Main; import soot.Scene; import soot.options.Options; +import tabby.common.utils.FileUtils; import tabby.config.GlobalConfiguration; import tabby.config.SootConfiguration; import tabby.core.collector.FileCollector; @@ -15,12 +15,10 @@ import tabby.core.container.RulesContainer; import tabby.core.scanner.CallGraphScanner; import tabby.core.scanner.ClassInfoScanner; -import tabby.common.utils.FileUtils; import java.io.File; import java.io.IOException; import java.util.*; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import static soot.SootClass.HIERARCHY; @@ -52,7 +50,7 @@ public void run() throws IOException { log.info("Get {} JDK dependencies", dependencies.size()); log.info("Try to collect all targets"); - Map cps = GlobalConfiguration.IS_EXCLUDE_JDK ? new HashMap<>():new HashMap<>(dependencies); + Map cps = new HashMap<>(dependencies); Map targets = new HashMap<>(); // 收集目标 GlobalConfiguration.rulesContainer = rulesContainer; @@ -87,6 +85,7 @@ public void runSootAnalysis(Map targets, List classpaths SootConfiguration.initSootOption(); addBasicClasses(); log.info("Load basic classes"); + Scene.v().setSootClassPath(String.join(File.pathSeparator, classpaths)); Scene.v().loadBasicClasses(); log.info("Load dynamic classes"); Scene.v().loadDynamicClasses(); diff --git a/src/main/java/tabby/core/collector/FileCollector.java b/src/main/java/tabby/core/collector/FileCollector.java index 2642aa7..ce32246 100644 --- a/src/main/java/tabby/core/collector/FileCollector.java +++ b/src/main/java/tabby/core/collector/FileCollector.java @@ -1,12 +1,12 @@ package tabby.core.collector; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import tabby.config.GlobalConfiguration; import tabby.analysis.data.FileLocation; import tabby.common.utils.FileUtils; -import tabby.common.utils.JavaVersionUtils; +import tabby.config.GlobalConfiguration; +import tabby.plugin.jmod.JModTransferPlugin; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -22,6 +22,9 @@ @Service public class FileCollector { + @Autowired + public JModTransferPlugin jModTransferPlugin; + public Map collect(String targetPath){ Map allTargets = new HashMap<>(); Path path = Paths.get(targetPath).toAbsolutePath(); @@ -50,36 +53,33 @@ public Map collect(String targetPath){ public Map collectJdkDependencies() throws IOException { - Map allJdkDependencies = new HashMap<>(); + Map dependencies = new HashMap<>(); - String javaHome = System.getProperty("java.home"); - if(JavaVersionUtils.isAtLeast(9)){ // jdk >= 9 + Set jdkLibs = null; + if(FileUtils.fileExists(GlobalConfiguration.JRE_LIBS_PATH)){ + jdkLibs = FileUtils.findAllJarFiles(GlobalConfiguration.JRE_LIBS_PATH, false); + } + if(jdkLibs == null || jdkLibs.isEmpty()){ + jdkLibs = FileUtils.findAllJdkDependencies(jModTransferPlugin); + } + + for(String filepath:jdkLibs){ if(GlobalConfiguration.IS_WITH_ALL_JDK){ - allJdkDependencies.putAll(FileUtils.findAllJdkDependencies(javaHome+"/jmods/", false)); - }else{ - String path = javaHome+"/jmods/java.base.jmod"; - File file = new File(path); - if(file.exists()){ - allJdkDependencies.put(FileUtils.getFileMD5(file), path); + dependencies.put(FileUtils.getFileMD5(filepath), filepath); + }else if(GlobalConfiguration.IS_JRE9_MODULE){ + if(filepath.endsWith("java.base.jmod.jar") + || filepath.endsWith("java.desktop.jmod.jar") + || filepath.endsWith("java.logging.jmod.jar")){ + dependencies.put(FileUtils.getFileMD5(filepath), filepath); } - } - }else{ // jdk <= 8 - if(GlobalConfiguration.IS_WITH_ALL_JDK){ - allJdkDependencies.putAll(FileUtils.findAllJdkDependencies(javaHome+"/lib", true)); - allJdkDependencies.putAll(FileUtils.findAllJdkDependencies(javaHome+"/../lib", false)); }else{ - String[] jre = new String[]{"lib/rt.jar","lib/jce.jar"}; - for(String cp:jre){ - String path = String.join(File.separator, javaHome, cp); - File file = new File(path); - if(file.exists()){ - allJdkDependencies.put(FileUtils.getFileMD5(file), path); - } + if(filepath.endsWith("rt.jar") || filepath.endsWith("jce.jar")){ + dependencies.put(FileUtils.getFileMD5(filepath), filepath); } } } - return allJdkDependencies; + return dependencies; } } diff --git a/src/main/java/tabby/plugin/jmod/JModTransferPlugin.java b/src/main/java/tabby/plugin/jmod/JModTransferPlugin.java new file mode 100644 index 0000000..fb3efa5 --- /dev/null +++ b/src/main/java/tabby/plugin/jmod/JModTransferPlugin.java @@ -0,0 +1,54 @@ +package tabby.plugin.jmod; + +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import tabby.common.utils.FileUtils; + +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.concurrent.CompletableFuture; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +/** + * @author wh1t3p1g + * @project tabby + * @since 2024/1/16 + */ +@Component +public class JModTransferPlugin { + + @Async("tabby-collector") + public CompletableFuture transfer(String source, String output){ + + try{ + if(source.endsWith(".jar")){ + FileUtils.copy(source, output); + }else if(source.endsWith(".jmod")){ + ZipFile moduleFile = new ZipFile(source); + ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(output)); + for(Enumeration enums = moduleFile.entries(); enums.hasMoreElements();){ + ZipEntry entry = (ZipEntry)enums.nextElement(); + String name = entry.getName(); + if((name.startsWith("classes") && !name.contains("module-info")) + || name.startsWith("resources") || name.startsWith("lib")){ + zipOutputStream.putNextEntry(new ZipEntry(name.substring(name.indexOf('/') + 1))); + InputStream in = moduleFile.getInputStream(entry); + while(in.available() > 0) + zipOutputStream.write(in.read()); + zipOutputStream.flush(); + } + } + zipOutputStream.close(); + } + } catch(Exception e) { + e.printStackTrace(); + } + + return CompletableFuture.completedFuture(true); + } + + +} diff --git a/src/main/java/tabby/plugin/JspCompilePlugin.java b/src/main/java/tabby/plugin/jsp/JspCompilePlugin.java similarity index 99% rename from src/main/java/tabby/plugin/JspCompilePlugin.java rename to src/main/java/tabby/plugin/jsp/JspCompilePlugin.java index 8876a97..e82958d 100644 --- a/src/main/java/tabby/plugin/JspCompilePlugin.java +++ b/src/main/java/tabby/plugin/jsp/JspCompilePlugin.java @@ -1,4 +1,4 @@ -package tabby.plugin; +package tabby.plugin.jsp; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt index 05d73d1..2187c78 100644 --- a/src/main/resources/banner.txt +++ b/src/main/resources/banner.txt @@ -5,6 +5,6 @@ |. | // __' \ (| _ \\ (| _ \\ / / \: | / / \\ \ |: |_) :)|: |_) :)/ / \__|(___/ \___)(_______/ (_______/|___/ - v1.2.0 + v1.3.0 Happy Hunting Bugs! XD https://github.com/wh1t3p1g/tabby