diff --git a/build.gradle b/build.gradle index a541cb5..51cb656 100644 --- a/build.gradle +++ b/build.gradle @@ -31,7 +31,8 @@ dependencies { // implementation 'org.springframework.boot:spring-boot-starter-data-neo4j' // implementation 'org.soot-oss:soot:4.2.1' // implementation 'org.soot-oss:soot:4.3.0-SNAPSHOT' - implementation 'org.soot-oss:soot:4.4.1' +// implementation 'org.soot-oss:soot:4.4.1' + implementation 'org.soot-oss:soot:4.5.0-SNAPSHOT' // implementation 'ca.mcgill.sable:soot:4.0.0' implementation 'com.google.code.gson:gson:2.10.1' diff --git a/config/settings.properties b/config/settings.properties index f14ce30..8e0b05a 100644 --- a/config/settings.properties +++ b/config/settings.properties @@ -6,6 +6,7 @@ tabby.output.directory = ./output/dev # debug tabby.debug.details = false +tabby.debug.print.current.methods = true # jdk settings tabby.build.useSettingJRE = false @@ -22,4 +23,7 @@ tabby.build.checkFatJar = true # pointed-to analysis tabby.build.isFullCallGraphCreate = false tabby.build.thread.timeout = 2 -tabby.build.isNeedToCreateIgnoreList = false \ No newline at end of file +tabby.build.method.timeout = 5 +tabby.build.isNeedToCreateIgnoreList = false +tabby.build.timeout.forceStop = false +tabby.build.isNeedToDealNewAddedMethod = true \ No newline at end of file diff --git a/rules/commonJars.json b/rules/commonJars.json index a573ffb..5f51fe3 100644 --- a/rules/commonJars.json +++ b/rules/commonJars.json @@ -1,21 +1,5 @@ [ - "commons-collections", - "commons-codec", - "commons-fileupload", - "commons-io", - "commons-configuration", - "commons-dbcp", - "commons-lang", - "commons-logging", - "commons-pool", - "commons-beanutils", - "commons-httpclient", - "commons-cli", - "commons-dbutils", - "commons-digester", - "commons-discovery", - "commons-email", - "commons-jxpath", + "commons-", "poi-", "jetty-", "apache-el", @@ -44,26 +28,22 @@ "crypto-", "ognl-", "logback-", - "taglibs-standard-", "zookeeper-", "jfreechart-", "jline-", - "xercesImpl-", "slf4j-api", "com.springsource.", - "sofa-runtime", "abatis-sqlmap", - "ojdbc14", + "ojdbc", "kryo-", "guice-", "xstream-", "struts", "xml-", "log4j-", - "zdal-", "aspectjweaver-", "oceanbase-", - "org.eclipse.osgi-", + "org.eclipse", "jboss-", "hibernate-", "castor-", @@ -82,6 +62,149 @@ "validation-api", "javax.servlet-", "jconsole-", - "xpp3_min", - "toolkit-common-" + "toolkit-common-", + "hessian-", + "antlr", + "axis2-", + "log4j-", + "jersey-", + "snakeyaml-", + "jakarta.annotation-", + "lombok-", + "xmlbeans-", + "velocity-", + "javax.servlet", + "freemarker-", + "slf4j-", + "dom4j-", + "c3p0-", + "standard-", + "bsh-", + "jstl-", + "tomcat-", + "jul-to-", + "jcl-to-", + "h2-", + "grpc-", + "java-jwt", + "okhttp-", + "annotations-", + "protobuf-", + "jedis-", + "lucene-", + "reflections-", + "htmlparser", + "xbean-", + "javax.mail", + "xmlsec-", + "java-uuid", + "xerces-", + "svg-", + "joda-", + "easy-okhttp", + "opensaml-", + "aspect", + "sms-", + "cglib-", + "xalan", + "org.apache.", + "ant.", + "graph-java-", + "mvel2-", + "jdom", + "google-oauth-", + "postgresql-", + "activemq-", + "flex-", + "swagger-", + "batik-", + "batik-", + "ehcache-", + "cssparser-", + "HikariCP-", + "pdfbox-", + "ibatis-", + "rhino-", + "cas-client-", + "dom4j.", + "itextpdf-", + "jna-", + "hutool-", + "javassist.", + "ooxml-", + "oracle.", + "quartz.", + "google-", + "jakarta-", + "dom.", + "org.restlet-", + "aws-sdk-", + "xerces.", + "log4j.", + "forms-", + "jni.", + "jmx.", + "mail-", + "activation", + "gson-", + "healthcheck-", + "notify-", + "ecj-", + "schema-sync-connector-", + "caffeine-", + "xfire-", + "avro-", + "javers-", + "sketch-jar", + "jsqlparser-", + "layout-", + "aviator-", + "classgraph-", + "jctools-", + "antlr4-", + "lookout", + "hadoop-", + "xmlbeans-", + "mvel2-", + "doom-", + "quartz-", + "snappy-", + "rxjava-", + "jedis-", + "lettuce-", + "spymemcached-", + "org.apache", + "org.springframework", + "freemarker-", + "wsdl4j-", + "lucene-", + "scheduler-", + "disaster-", + "rocketmq-", + "velocity-", + "jakarta.commons.", + "ram-", + "osgi-resource", + "sourceforge.", + "elasticsearch-", + "jsoup", + "basement-", + "script.juel", + "logger", + "umid-", + "mockito-", + "ibatis2-", + "jdbc.mysql", + "geronimo-", + "xmlschema-", + "axis-", + "http-agent-", + "javase-", + "smtp-", + "xmlpull-", + "openws-", + "reactor-", + "jcc-", + "byte-buddy", + "websocket-" ] \ No newline at end of file diff --git a/src/main/java/tabby/App.java b/src/main/java/tabby/App.java index 0164b40..9ca4cad 100644 --- a/src/main/java/tabby/App.java +++ b/src/main/java/tabby/App.java @@ -10,6 +10,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import sun.misc.Signal; import tabby.common.utils.FileUtils; @@ -18,6 +19,7 @@ @Slf4j @SpringBootApplication +@EnableScheduling @EntityScan({"tabby.common.bean"}) public class App { @@ -63,15 +65,21 @@ CommandLineRunner run(){ setLogDebugLevel(); Signal.handle(new Signal("INT"), // SIGINT signal -> { - log.error("Force Stop by control+c"); + log.error("Force Stop by Control + C"); stopThreads(GlobalConfiguration.tabbyCollectorExecutor); + stopThreads(GlobalConfiguration.tabbySaverExecutor); System.exit(0); }); analyser.run(); }catch (IllegalArgumentException e){ log.error(e.getMessage() + ", Please check your settings.properties file."); } - log.info("Done. Bye!"); + + if(GlobalConfiguration.GLOBAL_FORCE_STOP){ + log.info("OOM ERROR!"); + }else{ + log.info("Done. Bye!"); + } }; } diff --git a/src/main/java/tabby/analysis/PollutedVarsPointsToAnalysis.java b/src/main/java/tabby/analysis/PollutedVarsPointsToAnalysis.java index 2ae9a8f..beb3dd9 100644 --- a/src/main/java/tabby/analysis/PollutedVarsPointsToAnalysis.java +++ b/src/main/java/tabby/analysis/PollutedVarsPointsToAnalysis.java @@ -7,15 +7,15 @@ import soot.jimple.InstanceFieldRef; import soot.toolkits.graph.DirectedGraph; import soot.toolkits.scalar.ForwardFlowAnalysis; -import tabby.config.GlobalConfiguration; -import tabby.common.bean.ref.MethodReference; import tabby.analysis.data.Context; -import tabby.core.container.DataContainer; import tabby.analysis.data.TabbyVariable; import tabby.analysis.switcher.stmt.SimpleStmtSwitcher; import tabby.analysis.switcher.stmt.StmtSwitcher; import tabby.analysis.switcher.value.SimpleLeftValueSwitcher; import tabby.analysis.switcher.value.SimpleRightValueSwitcher; +import tabby.common.bean.ref.MethodReference; +import tabby.config.GlobalConfiguration; +import tabby.core.container.DataContainer; import java.util.ArrayList; import java.util.HashMap; @@ -38,6 +38,8 @@ public class PollutedVarsPointsToAnalysis extends ForwardFlowAnalysis in, Unit d, Map out) { - if(GlobalConfiguration.isNeedStop){ + if(GlobalConfiguration.isNeedStop || context.isAnalyseTimeout() || GlobalConfiguration.GLOBAL_FORCE_STOP){ + return; + } + + if(context.getMethodReference().isInitialed()){ + // 多个线程同时分析一个函数，且当前线程落后其他线程，则直接跳过后续的分析，且不保存当前分析所得的调用边 + isNormalExit = false; + return; + } + + if(context.isTimeout()){ // 如果当前函数分析超多最大限时，则停止分析当前函数 + context.setAnalyseTimeout(true); + isNormalExit = false; // 下一次会重新分析，这里先不保存call边 return; } @@ -135,6 +149,21 @@ protected void flowThrough(Map in, Unit d, Map newInitialFlow() { return new HashMap<>(emptyMap); @@ -198,6 +227,7 @@ public static PollutedVarsPointsToAnalysis makeDefault(MethodReference methodRef analysis.setMethodRef(methodRef); // 进行分析 analysis.doAnalysis(); + analysis.doEnd(); return analysis; } } diff --git a/src/main/java/tabby/analysis/data/Context.java b/src/main/java/tabby/analysis/data/Context.java index 893ec8b..771bc00 100644 --- a/src/main/java/tabby/analysis/data/Context.java +++ b/src/main/java/tabby/analysis/data/Context.java @@ -1,6 +1,7 @@ package tabby.analysis.data; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import soot.Local; import soot.SootField; import soot.SootFieldRef; @@ -8,11 +9,15 @@ import soot.jimple.InstanceFieldRef; import soot.jimple.StaticFieldRef; import tabby.common.bean.ref.MethodReference; +import tabby.config.GlobalConfiguration; +import java.time.Duration; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * 函数的域表示 @@ -21,8 +26,9 @@ * @author wh1t3P1g * @since 2020/11/24 */ +@Slf4j @Data -public class Context { +public class Context implements AutoCloseable { private String methodSignature; // 当前函数签名 private MethodReference methodReference; @@ -30,6 +36,7 @@ public class Context { private Local thisVar;// 设置当前的函数调用时的base变量是什么 或者说是this变量 private Map args = new HashMap<>(); // 前置函数的入参 private Context preContext;// 如果当前函数为被调用的函数，那么preContext指向之前的函数context + private Context subContext; private int depth; // 当前函数调用深度，限制无限循环的情况 // 经过flowThough 函数时，拷贝 in集合 private Map localMap; @@ -38,6 +45,13 @@ public class Context { // 用于return给当前 private TabbyVariable returnVar; private String topMethodSignature; + private boolean analyseTimeout = false; + private boolean isClosed = false; + + private LocalDateTime start; + private long startTime; + private long subTime; // 过程间分析，去除递归分析子函数所花的时间 + public Context(){ this.localMap = new HashMap<>(); @@ -50,12 +64,33 @@ public Context(String methodSignature, MethodReference methodReference, Context this.depth = depth; this.preContext = preContext; this.localMap = new HashMap<>(); + this.startTime = System.nanoTime(); + this.subTime = startTime; + this.start = LocalDateTime.now(); } public static Context newInstance(String methodSignature, MethodReference methodReference) { return new Context(methodSignature, methodReference,null,0); } + public boolean isTimeout(){ + long cost = TimeUnit.NANOSECONDS.toMinutes(System.nanoTime() - subTime); + boolean flag = cost >= GlobalConfiguration.METHOD_TIMEOUT; + if(flag && !analyseTimeout){ // 只打印最底层的timeout函数分析 + log.error("Method {} analysis timeout, cost {} min, plz check!", methodSignature, cost); + } + return flag; + } + + @Override + public void close(){ + // 清除引用 + methodReference = null; + preContext = null; + isClosed = true; + clear(); + } + /** * 创建一个子函数域 */ @@ -63,10 +98,32 @@ public Context createSubContext(String methodSignature, MethodReference methodRe Context subContext = new Context(methodSignature, methodReference, this,depth + 1); subContext.setGlobalMap(globalMap); // 同步所有globalmap subContext.setTopMethodSignature(topMethodSignature); + this.subContext = subContext; return subContext; } + public long getSeconds(){ + return Duration.between(start, LocalDateTime.now()).getSeconds(); + } + + public void sub(long cost){ + subTime += cost; + } + + public long cost(){ + return System.nanoTime() - startTime; + } + + public void setAnalyseTimeout(boolean analyseTimeout) { + this.analyseTimeout = analyseTimeout; + if(subContext != null && !subContext.isClosed()){ + subContext.setAnalyseTimeout(analyseTimeout); + } + } + public boolean isTopContext(){ + return preContext == null; + } /** * 接受 Local 和 staticField * 从localMap 和 globalMap 中分别查询对应的变量实例 diff --git a/src/main/java/tabby/analysis/model/CallEdgeBuilder.java b/src/main/java/tabby/analysis/model/CallEdgeBuilder.java new file mode 100644 index 0000000..be472fb --- /dev/null +++ b/src/main/java/tabby/analysis/model/CallEdgeBuilder.java @@ -0,0 +1,65 @@ +package tabby.analysis.model; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import soot.jimple.InvokeExpr; +import soot.jimple.Stmt; +import tabby.common.bean.ref.MethodReference; +import tabby.common.utils.SemanticUtils; +import tabby.core.container.DataContainer; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * call边builder + * 需要在分析当前invoke之前，把当前的所有信息保存起来 + * 所以在stmtSwitcher上用 + * @author wh1t3P1g + * @since 2022/2/17 + */ +@Slf4j +public class CallEdgeBuilder { + + private final LinkedList chains = new LinkedList<>(); + @Setter + private List pollutedPosition = new ArrayList<>(); + + public CallEdgeBuilder() { + chains.add(new IgnoreInvokeModel()); + chains.add(new DefaultInvokeModel()); + } + + public void build(Stmt stmt, + MethodReference caller, DataContainer dataContainer){ + try{ + + InvokeExpr ie = SemanticUtils.getInvokeExpr(stmt); + + if(ie == null){ + log.error("get invoke expr error: {}", stmt); + return; + } + + MethodReference callee = dataContainer.getOrAddMethodRef(ie); + + if(callee == null){ + log.error("get callee error: {}", stmt); + return; + } + + for(Model model:chains){ + model.setPP(pollutedPosition); + if(model.apply(stmt, false, caller, callee, dataContainer)){ + break; + } + } + } catch (Exception e){ + // 多线程的情况下 可能在soot提取method上会有一些问题 +// log.error(e.getMessage()); +// e.printStackTrace(); + } + + } +} diff --git a/src/main/java/tabby/analysis/model/DefaultInvokeModel.java b/src/main/java/tabby/analysis/model/DefaultInvokeModel.java index de1f093..5484b08 100644 --- a/src/main/java/tabby/analysis/model/DefaultInvokeModel.java +++ b/src/main/java/tabby/analysis/model/DefaultInvokeModel.java @@ -6,9 +6,9 @@ import soot.ValueBox; import soot.jimple.*; import soot.jimple.internal.JimpleLocalBox; -import tabby.core.container.DataContainer; import tabby.common.bean.edge.Call; import tabby.common.bean.ref.MethodReference; +import tabby.core.container.DataContainer; import java.util.ArrayList; import java.util.Arrays; @@ -20,8 +20,9 @@ */ @Getter @Setter -public class DefaultInvokeModel{ +public class DefaultInvokeModel implements Model{ + private List pollutedPosition = new ArrayList<>(); private static final List IGNORE_LIST = new ArrayList<>(Arrays.asList( "()>", "", @@ -33,32 +34,33 @@ public class DefaultInvokeModel{ "" )); + @Override + public void setPP(List pollutedPosition) { + this.pollutedPosition = pollutedPosition; + } - public void apply(Stmt stmt, boolean isManual, MethodReference methodRef, - MethodReference targetMethodRef, DataContainer dataContainer) { - - String signature = targetMethodRef.getSignature(); - if(IGNORE_LIST.contains(signature)) return ; - // 剔除递归调用自身的情况 - if(methodRef.getId().equals(targetMethodRef.getId())) return ; + public boolean apply(Stmt stmt, boolean isManual, MethodReference caller, MethodReference callee, DataContainer dataContainer) { + if(caller.getId().equals(callee.getId())) return false; InvokeExpr ie = stmt.getInvokeExpr(); - - Call call = Call.newInstance(methodRef, targetMethodRef); + Call call = Call.newInstance(caller, callee); if(isManual){ - call.setRealCallType(targetMethodRef.getClassname()); + call.setRealCallType(callee.getClassname()); call.setInvokerType("ManualInvoke"); }else{ - call.setRealCallType(getRealCallType(ie, targetMethodRef)); + call.setRealCallType(getRealCallType(ie, callee)); call.setInvokerType(getInvokeType(ie)); } + + call.setPollutedPosition(new ArrayList<>(pollutedPosition)); call.setLineNum(stmt.getJavaSourceStartLineNumber()); call.generateId(); - if(!methodRef.getCallEdge().contains(call)){ - methodRef.getCallEdge().add(call); + if(!caller.getCallEdge().contains(call)){ + caller.getCallEdge().add(call); dataContainer.store(call); } + return true; } public String getInvokeType(InvokeExpr ie){ @@ -95,4 +97,6 @@ public String getRealCallType(InvokeExpr ie, MethodReference targetMethodRef){ return classname; } + + } diff --git a/src/main/java/tabby/analysis/model/IgnoreInvokeModel.java b/src/main/java/tabby/analysis/model/IgnoreInvokeModel.java new file mode 100644 index 0000000..7934508 --- /dev/null +++ b/src/main/java/tabby/analysis/model/IgnoreInvokeModel.java @@ -0,0 +1,106 @@ +package tabby.analysis.model; + +import soot.jimple.Stmt; +import tabby.common.bean.ref.MethodReference; +import tabby.config.GlobalConfiguration; +import tabby.core.container.DataContainer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author wh1t3p1g + * @since 2021/9/1 + */ +public class IgnoreInvokeModel extends DefaultInvokeModel { + + private static final List IGNORE_LIST = new ArrayList<>(Arrays.asList( + "()>", + "", + "", + "", + "", + "", + "", + "" +// "" + )); + + private static final List WEB_MODE_IGNORE_LIST = new ArrayList<>(Arrays.asList( + "java.lang.String toString()", + "int hashCode()", + "void close()", + "void flush()", + "int length()", + "java.lang.Class getClass()" +// "boolean equals(java.lang.Object)" + )); + + private static final List EXCLUDE_LIST = new ArrayList<>(Arrays.asList( + "contains", + "equals" + )); + + private static final List WEB_MODE_IGNORE_CLASSNAME_LIST = new ArrayList<>(Arrays.asList( + "java.util.List", + "java.util.ArrayList", + "java.util.CopyOnWriteArrayList", + "java.util.LinkedList", + "java.util.Stack", + "java.util.Vector", + "java.util.Set", + "java.util.LinkedHashSet", + "java.util.TreeSet", + "java.util.HashSet", + "java.util.Collection", + "java.util.Queue", + "java.util.AbstractQueue", + "java.util.PriorityQueue", + "java.util.ArrayDeque", + "java.util.HashTable", + "java.util.concurrent.CopyOnWriteArraySet", + "java.util.Map", + "java.util.HashMap", + "java.util.TreeMap", + "java.util.LinkedHashMap", + "java.util.WeakHashMap", + "java.util.SortedMap", + "java.util.concurrent.ConcurrentHashMap", + "java.util.concurrent.ConcurrentMap", + "java.util.Arrays", + "java.util.Properties", + "java.util.Iterator", + "java.util.Enumeration", + "java.io.PrintStream", + "java.lang.String", + "java.lang.StringBuilder" + )); + + @Override + public boolean apply(Stmt stmt, boolean isManual, MethodReference caller, MethodReference callee, DataContainer dataContainer) { + String signature = callee.getSignature(); + if(IGNORE_LIST.contains(signature)){ + return true; + } + + if(GlobalConfiguration.IS_WEB_MODE){ + String name = callee.getName(); + if(EXCLUDE_LIST.contains(name)){ + return false; + } + + String subSignature = callee.getSubSignature(); + if(WEB_MODE_IGNORE_LIST.contains(subSignature)){ + return true; + } + + String classname = callee.getClassname(); + if(WEB_MODE_IGNORE_CLASSNAME_LIST.contains(classname)){ + return true; + } + } + + return false; + } +} diff --git a/src/main/java/tabby/analysis/model/Model.java b/src/main/java/tabby/analysis/model/Model.java new file mode 100644 index 0000000..ced0352 --- /dev/null +++ b/src/main/java/tabby/analysis/model/Model.java @@ -0,0 +1,51 @@ +package tabby.analysis.model; + +import soot.SootClass; +import soot.Type; +import soot.Value; +import soot.jimple.InvokeExpr; +import soot.jimple.Stmt; +import tabby.common.bean.ref.MethodReference; +import tabby.common.utils.SemanticUtils; +import tabby.core.container.DataContainer; + +import java.util.List; +import java.util.Set; + +/** + * @author wh1t3p1g + * @since 2021/8/31 + */ +public interface Model { + + boolean apply(Stmt stmt, boolean isManual, MethodReference methodRef, + MethodReference targetMethodRef, DataContainer dataContainer); + + void setPP(List pollutedPosition); + + default Type getArgType(InvokeExpr ie, int index){ + try{ + Value arg = ie.getArg(index); + if(arg != null) { + return arg.getType(); + } + }catch (Exception e){ + // class not found + } + return null; + } + + default boolean isAssignableFrom(String target, Class cls, DataContainer dataContainer){ + if(target.equals(cls.getName())) return true; + + try{ + SootClass targetClass = SemanticUtils.getSootClass(target); + Set allFatherNodes = SemanticUtils.getAllFatherNodes(targetClass, true); + return allFatherNodes.contains(cls.getName()); + }catch (Exception ignored){ + } + + return false; + } + +} diff --git a/src/main/java/tabby/analysis/switcher/InvokeExprSwitcher.java b/src/main/java/tabby/analysis/switcher/InvokeExprSwitcher.java index e5754be..8036b3a 100644 --- a/src/main/java/tabby/analysis/switcher/InvokeExprSwitcher.java +++ b/src/main/java/tabby/analysis/switcher/InvokeExprSwitcher.java @@ -6,13 +6,13 @@ import soot.*; import soot.jimple.*; import soot.jimple.internal.JimpleLocal; -import tabby.core.container.DataContainer; -import tabby.core.container.RulesContainer; -import tabby.analysis.data.TabbyVariable; import tabby.analysis.PollutedVarsPointsToAnalysis; -import tabby.common.bean.edge.Call; +import tabby.analysis.data.TabbyVariable; +import tabby.analysis.model.CallEdgeBuilder; import tabby.common.bean.ref.MethodReference; import tabby.common.utils.PositionUtils; +import tabby.core.container.DataContainer; +import tabby.core.container.RulesContainer; import java.util.*; @@ -38,23 +38,22 @@ public class InvokeExprSwitcher extends AbstractJimpleValueSwitch { private DataContainer dataContainer; private RulesContainer rulesContainer; + private CallEdgeBuilder builder = new CallEdgeBuilder(); @Override public void caseStaticInvokeExpr(StaticInvokeExpr v) { if(isNecessaryEdge("StaticInvoke", v)){ - SootMethodRef sootMethodRef = v.getMethodRef(); generate(v); - buildCallRelationship(sootMethodRef.getDeclaringClass().getName(), sootMethodRef, "StaticInvoke"); + buildCallRelationship(v); } } @Override public void caseVirtualInvokeExpr(VirtualInvokeExpr v) { // a.A() - SootMethodRef sootMethodRef = v.getMethodRef(); baseValue = v.getBase(); generate(v); - buildCallRelationship(v.getBase().getType().toString(), sootMethodRef, "VirtualInvoke"); + buildCallRelationship(v); } @Override @@ -63,57 +62,19 @@ public void caseSpecialInvokeExpr(SpecialInvokeExpr v) {// 初始化 this.xxx() if(sootMethodRef.getSignature().contains("") && v.getArgCount() == 0) return; // 无参的构造函数 不影响数据流分析 baseValue = v.getBase(); generate(v); - buildCallRelationship(v.getBase().getType().toString(), sootMethodRef, "SpecialInvoke"); + buildCallRelationship(v); } @Override public void caseInterfaceInvokeExpr(InterfaceInvokeExpr v) { - SootMethodRef sootMethodRef = v.getMethodRef(); baseValue = v.getBase(); generate(v); - buildCallRelationship(v.getBase().getType().toString(), sootMethodRef, "InterfaceInvoke"); + buildCallRelationship(v); } - public void buildCallRelationship(String classname, SootMethodRef sootMethodRef, String invokerType){ - MethodReference target = dataContainer.getOrAddMethodRef(sootMethodRef, sootMethodRef.resolve());// 递归父类，接口 查找目标函数 - MethodReference source = dataContainer.getMethodRefBySignature(this.source.getClassname(), this.source.getSignature()); - - if(target.isSink()){ - // 调用sink函数时，需要符合sink函数的可控点，如果均为可控点，则当前调用是可控的 - for(int i:target.getPollutedPosition()){ - if(pollutedPosition.size() > i+1 && pollutedPosition.get(i+1) == PositionUtils.NOT_POLLUTED_POSITION){ - isPolluted = false; - break; - } - } - } - - if(source != null - && !target.isIgnore() - && isPolluted){ // 剔除不可控边 - - if("java.lang.String".equals(classname) // 这种情况一般均不可控，可控也没有意义 - && ("equals".equals(target.getName()) - || "hashCode".equals(target.getName()) - || "length".equals(target.getName()))) return; - - if("java.lang.StringBuilder".equals(classname) // 这种情况一般均不可控，可控也没有意义 - && ("toString".equals(target.getName()) - || "hashCode".equals(target.getName()))) return; - - Call call = Call.newInstance(source, target); - call.setRealCallType(classname); - call.setInvokerType(invokerType); - call.setPollutedPosition(new ArrayList<>(pollutedPosition)); - call.setLineNum(unit.getJavaSourceStartLineNumber()); - call.generateId(); - - if(!source.getCallEdge().contains(call)){ - source.getCallEdge().add(call); - dataContainer.store(call); - } - - } + public void buildCallRelationship(InvokeExpr ie){ + builder.setPollutedPosition(pollutedPosition); + builder.build((Stmt) ie, source, dataContainer); } public boolean isNecessaryEdge(String type, T v){ diff --git a/src/main/java/tabby/analysis/switcher/Switcher.java b/src/main/java/tabby/analysis/switcher/Switcher.java index 69346f5..e88ff4d 100644 --- a/src/main/java/tabby/analysis/switcher/Switcher.java +++ b/src/main/java/tabby/analysis/switcher/Switcher.java @@ -38,7 +38,7 @@ public class Switcher { * @param method * @param methodRef */ - public static PollutedVarsPointsToAnalysis doMethodAnalysis(Context context, + public static boolean doMethodAnalysis(Context context, DataContainer dataContainer, SootMethod method, MethodReference methodRef){ @@ -47,28 +47,51 @@ public static PollutedVarsPointsToAnalysis doMethodAnalysis(Context context, || method.isPhantom()){ methodRef.setInitialed(true); methodRef.setActionInitialed(true); - return null; + return false; } if(methodRef.isActionInitialed() && methodRef.isInitialed()){ // 已经初始化过了 - return null; + return false; + } + + if(methodRef.isBodyParseError()) return false; + + int maxSleepTimes = 5; + while(methodRef.isRunning() && maxSleepTimes > 0){ + // 如果已经有线程在运行了，随机睡几秒 + int random = (int)(Math.random()*20); + try { + Thread.sleep(random); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + // 如果睡完，已经分析完了，则直接返回 + if(methodRef.isActionInitialed()){ + return true; + } + maxSleepTimes--; } JimpleBody body = (JimpleBody) SemanticUtils.retrieveBody(method, methodRef.getSignature(), true); - if(body == null) return null; + if(body == null) return false; + + methodRef.setRunning(true); UnitGraph graph = new BriefUnitGraph(body); - PollutedVarsPointsToAnalysis pta = - PollutedVarsPointsToAnalysis - .makeDefault(methodRef, body, graph, - dataContainer, context, !methodRef.isActionInitialed()); + PollutedVarsPointsToAnalysis.makeDefault(methodRef, body, graph, dataContainer, context, !methodRef.isActionInitialed()); methodRef.setInitialed(true); methodRef.setActionInitialed(true); - return pta; + return true; }catch (Exception e){ + String msg = e.getMessage(); + if(msg != null && msg.contains("Body retrieve error")){ + methodRef.setBodyParseError(true); + } throw new RuntimeException(e); + }finally { + methodRef.setRunning(false); } } @@ -106,8 +129,7 @@ public static TabbyVariable doInvokeExprAnalysis( SootClass cls = invokeExpr.getMethod().getDeclaringClass(); SootMethod invokedMethod = invokeExpr.getMethod(); - MethodReference methodRef = dataContainer - .getOrAddMethodRef(invokeExpr.getMethodRef(), invokedMethod); + MethodReference methodRef = dataContainer.getOrAddMethodRef(invokedMethod); // construct call edge String invokeType = ""; @@ -128,8 +150,11 @@ public static TabbyVariable doInvokeExprAnalysis( // 由于获取到的method是没有函数内容的，所以需要找到对应的具体实现来进行分析 // 这里继续进行简化，对于无返回的函数调用，可以仍然保持原状，也就是舍弃了函数参数在函数体内可能发生的变化 // 对于有返回的函数调用，则找到一个会影响返回值的具体实现 - Context subContext = context.createSubContext(methodRef.getSignature(), methodRef); - Switcher.doMethodAnalysis(subContext, dataContainer, invokedMethod, methodRef); + + try(Context subContext = context.createSubContext(methodRef.getSignature(), methodRef)){ + Switcher.doMethodAnalysis(subContext, dataContainer, invokedMethod, methodRef); + context.sub(subContext.cost()); + } } // 回溯 TabbyVariable retVar = null; diff --git a/src/main/java/tabby/common/bean/ref/MethodReference.java b/src/main/java/tabby/common/bean/ref/MethodReference.java index de48816..221ad07 100644 --- a/src/main/java/tabby/common/bean/ref/MethodReference.java +++ b/src/main/java/tabby/common/bean/ref/MethodReference.java @@ -17,6 +17,8 @@ import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; /** * @author wh1t3P1g @@ -82,7 +84,8 @@ public class MethodReference { private transient boolean isPreCollected = false; private transient boolean isContainSomeError = false; - + private transient AtomicBoolean isRunning = new AtomicBoolean(false); + private transient AtomicInteger timeoutTimes = new AtomicInteger(0); /** * 污染传递点，主要标记2种类型，this和param * 其他函数可以依靠relatedPosition，来判断当前位置是否是通路 @@ -179,6 +182,22 @@ public void addAction(String key, String value){ actions.put(key, value); } + public boolean isEverTimeout(){ + return timeoutTimes.get() >= 3; + } + + public void incrementTimeoutTimes(){ + timeoutTimes.incrementAndGet(); + } + + public void setRunning(boolean flag){ + isRunning.set(flag); + } + + public boolean isRunning(){ + return isRunning.get(); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/tabby/common/utils/SemanticUtils.java b/src/main/java/tabby/common/utils/SemanticUtils.java index 4fd727d..7fdbd57 100644 --- a/src/main/java/tabby/common/utils/SemanticUtils.java +++ b/src/main/java/tabby/common/utils/SemanticUtils.java @@ -1,12 +1,8 @@ package tabby.common.utils; -import com.google.common.collect.ImmutableMap; import lombok.extern.slf4j.Slf4j; import soot.*; -import soot.jimple.ArrayRef; -import soot.jimple.Constant; -import soot.jimple.FieldRef; -import soot.jimple.InstanceFieldRef; +import soot.jimple.*; import soot.tagkit.*; import tabby.config.GlobalConfiguration; @@ -104,6 +100,50 @@ public static SootMethod getMethod(SootClass cls, String subSignature){ } } + public static SootMethod getMethod(InvokeExpr ie) { + SootMethod method = null; + int i = 0; + do{ + try{ + method = ie.getMethod(); + }catch (Exception e){ + // try again + int random = (int)(Math.random()*100); + try { + Thread.sleep(random); + } catch (InterruptedException ex) { + System.exit(0); + } + } + i++; + // 防止因为多线程导致invoke重复提取 + }while(method == null && i < 3); + + return method; + } + + public static SootMethod getMethod(SootMethodRef sootMethodRef) { + SootMethod method = null; + int i = 0; + do{ + try{ + method = sootMethodRef.resolve(); + }catch (Exception e){ + // try again + int random = (int)(Math.random()*100); + try { + Thread.sleep(random); + } catch (InterruptedException ex) { + System.exit(0); + } + } + i++; + // 防止因为多线程导致invoke重复提取 + }while(method == null && i < 3); + + return method; + } + public static SootMethod getMethod(String classname, String subSignature){ SootClass cls = getSootClass(classname); @@ -112,6 +152,24 @@ public static SootMethod getMethod(String classname, String subSignature){ return getMethod(cls, subSignature); } + public static InvokeExpr getInvokeExpr(Stmt stmt) throws InterruptedException { + InvokeExpr ie = null; + int i = 0; + do{ + try{ + ie = stmt.getInvokeExpr(); + }catch (Exception e){ + // try again + int random = (int)(Math.random()*100); + Thread.sleep(random); + } + i++; + // 防止因为多线程导致invoke重复提取 + }while(ie == null && i < 3); + + return ie; + } + public static synchronized Body retrieveBody(SootMethod method, String signature, boolean tries){ ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -127,7 +185,7 @@ public static synchronized Body retrieveBody(SootMethod method, String signature } catch (ExecutionException | InterruptedException e) { // e.printStackTrace(); String msg = e.getMessage(); - if(tries && msg.contains("Failed to convert")){ + if(tries && msg != null && msg.contains("Failed to convert")){ Throwable temp = e; do{ String message = temp.getMessage(); diff --git a/src/main/java/tabby/common/utils/TickTock.java b/src/main/java/tabby/common/utils/TickTock.java index 601956a..525415d 100644 --- a/src/main/java/tabby/common/utils/TickTock.java +++ b/src/main/java/tabby/common/utils/TickTock.java @@ -3,6 +3,7 @@ import lombok.extern.slf4j.Slf4j; import tabby.config.GlobalConfiguration; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -17,7 +18,7 @@ public class TickTock { private int split; private boolean show = false; private CountDownLatch latch; - + private long startTime; public TickTock(int total, boolean show) { this.total = total; @@ -27,6 +28,8 @@ public TickTock(int total, boolean show) { if(this.split == 0){ this.split = 1; } + GlobalConfiguration.tickTock = this; + startTime = System.nanoTime(); } public void await() { @@ -61,13 +64,12 @@ public void error(String msg, Object... objs){ } } - public void awaitWithoutTimeout(){ + public void awaitUntilCompleted(){ try { - info("Waiting for {} classes to be collected...", latch.getCount()); latch.await(); } catch (InterruptedException e) { - e.printStackTrace(); - error("Still have {} classes to collected.", latch.getCount()); +// e.printStackTrace(); + error("Still have {} methods to analysis, but it reached the max timeout.", latch.getCount()); } } @@ -84,7 +86,24 @@ public void ticktock(){ long remain = latch.getCount(); long finished = total - remain; if(finished % split == 0 || finished == total){ - info("Status: {}%, Remain: {}", String.format("%.1f",finished*0.1/total*1000), remain); + info("Status: {}%, Finished: {}, Remain: {}, Cost: {} Mins", + String.format("%.1f",finished*0.1/total*1000), finished, remain, + TimeUnit.NANOSECONDS.toMinutes(System.nanoTime() - startTime)); + if(finished == total){ + info("All tasks completed."); + } + } + } + + public void ticktockForScheduleTask(Map map){ + long remain = latch.getCount(); + long finished = total - remain; + int size = 0; + if(map != null){ + size = map.size(); } + info("Status: {}%, Finished: {}, Remain: {}, Cost: {} Mins, Current: {}", + String.format("%.1f",finished*0.1/total*1000), finished, remain, + TimeUnit.NANOSECONDS.toMinutes(System.nanoTime() - startTime), size); } } diff --git a/src/main/java/tabby/config/AsyncConfiguration.java b/src/main/java/tabby/config/AsyncConfiguration.java index 74c8312..6ebd46a 100644 --- a/src/main/java/tabby/config/AsyncConfiguration.java +++ b/src/main/java/tabby/config/AsyncConfiguration.java @@ -9,6 +9,9 @@ import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; +import static java.lang.Math.max; +import static java.lang.Math.min; + /** * @author wh1t3P1g * @since 2021/4/23 @@ -29,6 +32,16 @@ public Executor master() { return executor; } + @Bean("tabby-saver") + public Executor saver() { + int poolSize = min(max(CORE_POOL_SIZE/2, 2), 4); + int corePoolSize = poolSize+1; + int maxPoolSize = poolSize+2; + ThreadPoolTaskExecutor executor = makeExecutor(corePoolSize, maxPoolSize, "tabby-saver"); + GlobalConfiguration.tabbySaverExecutor = executor; + return executor; + } + private ThreadPoolTaskExecutor makeExecutor(int corePoolSize, int maxPoolSize, String prefix){ log.info("Open {} size for thread pool {}", corePoolSize, prefix); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); @@ -36,8 +49,8 @@ private ThreadPoolTaskExecutor makeExecutor(int corePoolSize, int maxPoolSize, S executor.setMaxPoolSize(maxPoolSize); executor.setQueueCapacity(maxPoolSize * 1000); executor.setKeepAliveSeconds(300); - executor.setAwaitTerminationSeconds(1); executor.setThreadNamePrefix(prefix+"-"); + executor.setAwaitTerminationSeconds(1); executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); executor.initialize(); return executor; diff --git a/src/main/java/tabby/config/GlobalConfiguration.java b/src/main/java/tabby/config/GlobalConfiguration.java index fd432f4..ab21403 100644 --- a/src/main/java/tabby/config/GlobalConfiguration.java +++ b/src/main/java/tabby/config/GlobalConfiguration.java @@ -4,6 +4,7 @@ import com.google.gson.GsonBuilder; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import tabby.common.utils.TickTock; import tabby.core.container.RulesContainer; import tabby.common.utils.FileUtils; @@ -62,9 +63,18 @@ public class GlobalConfiguration { public static boolean isNeedStop = false; public static boolean IS_JRE9_MODULE = false; public static boolean IS_USING_SETTING_JRE = false; + public static boolean TIMEOUT_FORCE_STOP = false; + public static boolean PRINT_METHODS = false; + public static boolean IS_NEED_TO_DEAL_NEW_ADDED_METHOD = true; + public static boolean IS_NEED_ADD_TO_TIMEOUT_LIST = true; + public static int METHOD_TIMEOUT = 10; + public static long METHOD_TIMEOUT_SECONDS = 600; public static String TARGET_JAVA_HOME = null; public static String THREAD_POOL_SIZE = "max"; public static ThreadPoolTaskExecutor tabbyCollectorExecutor; + public static ThreadPoolTaskExecutor tabbySaverExecutor; + public static TickTock tickTock; + public static boolean GLOBAL_FORCE_STOP = false; public static void init(){ if(props == null){ @@ -150,12 +160,21 @@ public static void initConfig(){ IS_CHECK_FAT_JAR = getBooleanProperty("tabby.build.checkFatJar", "false", props); IS_FULL_CALL_GRAPH_CONSTRUCT = getBooleanProperty("tabby.build.isFullCallGraphCreate", "false", props); IS_NEED_TO_CREATE_IGNORE_LIST = getBooleanProperty("tabby.build.isNeedToCreateIgnoreList", "true", props); + TIMEOUT_FORCE_STOP = getBooleanProperty("tabby.build.timeout.forceStop", "true", props); + PRINT_METHODS = getBooleanProperty("tabby.debug.print.current.methods", "false", props); + IS_NEED_TO_DEAL_NEW_ADDED_METHOD = getBooleanProperty("tabby.build.isNeedToDealNewAddedMethod", "true", props); try{ TIMEOUT = getIntProperty("tabby.build.thread.timeout", "2", props); }catch (Exception ignore){ } + try{ + METHOD_TIMEOUT = getIntProperty("tabby.build.method.timeout", "10", props); + METHOD_TIMEOUT_SECONDS = METHOD_TIMEOUT * 60L; + }catch (Exception ignore){ + } + // 支持绝对路径 issue 7 if(!IS_JDK_ONLY && TARGET != null && !FileUtils.fileExists(TARGET)){ String target = String.join(File.separator, System.getProperty("user.dir"), TARGET); diff --git a/src/main/java/tabby/core/Analyser.java b/src/main/java/tabby/core/Analyser.java index ecb8a42..ab3f7c9 100644 --- a/src/main/java/tabby/core/Analyser.java +++ b/src/main/java/tabby/core/Analyser.java @@ -1,12 +1,16 @@ package tabby.core; +import com.google.common.collect.ImmutableSet; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import soot.CompilationDeathException; import soot.Main; import soot.Scene; import soot.options.Options; +import tabby.analysis.data.Context; import tabby.common.utils.FileUtils; import tabby.config.GlobalConfiguration; import tabby.config.SootConfiguration; @@ -18,6 +22,8 @@ import java.io.File; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.TimeUnit; @@ -55,6 +61,7 @@ public void run() throws IOException { // 收集目标 GlobalConfiguration.rulesContainer = rulesContainer; if(!GlobalConfiguration.IS_JDK_ONLY){ + log.info("Target: {}", GlobalConfiguration.TARGET); Map files = fileCollector.collect(GlobalConfiguration.TARGET); cps.putAll(files); targets.putAll(files); @@ -76,8 +83,13 @@ public void run() throws IOException { } runSootAnalysis(targets, new ArrayList<>(cps.values())); - dataContainer.count(); - dataContainer.save2CSV(); + + if(!GlobalConfiguration.GLOBAL_FORCE_STOP){ + // 仅当OOM未发生时，保存当前结果到CSV文件 + dataContainer.count(); + // output + dataContainer.save2CSV(); + } } public void runSootAnalysis(Map targets, List classpaths){ @@ -144,4 +156,41 @@ public void addBasicClasses(){ } } + @Async("tabby-saver") + @Scheduled(fixedRate = 2, timeUnit = TimeUnit.MINUTES) + public void count(){ + if(GlobalConfiguration.tickTock != null){ + GlobalConfiguration.tickTock.ticktockForScheduleTask(dataContainer.getRunningMethods()); + // print current running methods and kill overtime methods + if(GlobalConfiguration.TIMEOUT_FORCE_STOP){ + if(dataContainer.getRunningMethods() == null || dataContainer.getRunningMethods().isEmpty()) return; + Set uuids = ImmutableSet.copyOf(dataContainer.getRunningMethods().keySet()); + if(GlobalConfiguration.PRINT_METHODS) { + LocalDateTime now = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + log.warn("================================{}================================", formatter.format(now)); + } + for(String uuid: uuids){ + Context context = dataContainer.getRunningMethodContext(uuid); + if(context == null) continue; + String method = context.getMethodSignature(); + long cost = context.getSeconds(); + if(cost >= GlobalConfiguration.METHOD_TIMEOUT_SECONDS) { + context.setAnalyseTimeout(true); + if(GlobalConfiguration.PRINT_METHODS){ + log.warn("Cost {}s, trigger killer, {}", String.format("%5d",context.getSeconds()), method); + } + }else{ + if(GlobalConfiguration.PRINT_METHODS){ + log.warn("Cost {}s, {}", String.format("%5d",context.getSeconds()), method); + } + } + } + if(GlobalConfiguration.PRINT_METHODS) { + log.warn("================================end================================"); + } + } + } + } + } \ No newline at end of file diff --git a/src/main/java/tabby/core/collector/CallEdgeCollector.java b/src/main/java/tabby/core/collector/CallEdgeCollector.java index 0fdf76e..3c28b36 100644 --- a/src/main/java/tabby/core/collector/CallEdgeCollector.java +++ b/src/main/java/tabby/core/collector/CallEdgeCollector.java @@ -6,14 +6,13 @@ import soot.Modifier; import soot.SootMethod; import soot.Unit; -import soot.jimple.InvokeExpr; import soot.jimple.JimpleBody; import soot.jimple.Stmt; -import tabby.common.utils.SemanticUtils; -import tabby.core.container.DataContainer; -import tabby.analysis.model.DefaultInvokeModel; +import tabby.analysis.model.CallEdgeBuilder; import tabby.common.bean.ref.MethodReference; +import tabby.common.utils.SemanticUtils; import tabby.common.utils.TickTock; +import tabby.core.container.DataContainer; /** * @author wh1t3p1g @@ -59,15 +58,11 @@ public void collect(MethodReference methodRef, DataContainer dataContainer, Tick return; } - DefaultInvokeModel model = new DefaultInvokeModel(); + CallEdgeBuilder builder = new CallEdgeBuilder(); for(Unit unit:body.getUnits()){ Stmt stmt = (Stmt) unit; if(stmt.containsInvokeExpr()){ - InvokeExpr ie = stmt.getInvokeExpr(); - SootMethod targetMethod = ie.getMethod(); - MethodReference targetMethodRef - = dataContainer.getOrAddMethodRef(ie.getMethodRef(), targetMethod); - model.apply(stmt, false, methodRef, targetMethodRef, dataContainer); + builder.build(stmt, methodRef, dataContainer); } } }catch (RuntimeException e){ diff --git a/src/main/java/tabby/core/collector/CallGraphCollector.java b/src/main/java/tabby/core/collector/CallGraphCollector.java index 7501ecd..1cc8178 100644 --- a/src/main/java/tabby/core/collector/CallGraphCollector.java +++ b/src/main/java/tabby/core/collector/CallGraphCollector.java @@ -2,6 +2,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import soot.Modifier; import soot.SootMethod; @@ -9,8 +10,11 @@ import tabby.analysis.switcher.Switcher; import tabby.common.bean.ref.MethodReference; import tabby.common.utils.TickTock; +import tabby.config.GlobalConfiguration; import tabby.core.container.DataContainer; +import java.util.UUID; + /** * @author wh1t3P1g * @since 2021/4/23 @@ -20,8 +24,9 @@ @Setter public class CallGraphCollector { -// @Async("tabby-collector") + @Async("tabby-collector") public void collect(MethodReference methodRef, DataContainer dataContainer, TickTock tickTock){ + String uuid = UUID.randomUUID().toString(); try{ SootMethod method = methodRef.getMethod(); if(method == null) { @@ -51,22 +56,33 @@ public void collect(MethodReference methodRef, DataContainer dataContainer, Tick // } log.debug(method.getDeclaringClass().getName()+" "+method.getName()); // TODO debug - - Context context = Context.newInstance(method.getSignature(), methodRef); - - Switcher.doMethodAnalysis(context, dataContainer, method, methodRef); - context.clear(); + try(Context context = Context.newInstance(method.getSignature(), methodRef)){ + dataContainer.getRunningMethods().put(uuid, context); + Switcher.doMethodAnalysis(context, dataContainer, method, methodRef); + } }catch (RuntimeException e){ - log.error("Something error on call graph. " + methodRef.getSignature()); String msg = e.getMessage(); - log.error(msg); - }catch (Exception e){ + if(msg != null && msg.contains("Body retrieve error")){ + log.warn("Body retrieve error: " + methodRef.getSignature()); + }else{ + log.error(msg); + e.printStackTrace(); + } + } + catch (OutOfMemoryError e){ + log.error("OOM Error!!!! Force Stop Everything!!!"); + GlobalConfiguration.GLOBAL_FORCE_STOP = true; + } + catch (Exception e){ if(e instanceof InterruptedException) { log.error("Thread interrupted. " + methodRef.getSignature()); } else { log.error("Something error on call graph. "+methodRef.getSignature()); e.printStackTrace(); } + }finally { + dataContainer.getRunningMethods().remove(uuid); + methodRef.setRunning(false); } tickTock.countDown(); } diff --git a/src/main/java/tabby/core/collector/ClassInfoCollector.java b/src/main/java/tabby/core/collector/ClassInfoCollector.java index 1fd756a..a9afab3 100644 --- a/src/main/java/tabby/core/collector/ClassInfoCollector.java +++ b/src/main/java/tabby/core/collector/ClassInfoCollector.java @@ -5,16 +5,20 @@ import org.springframework.stereotype.Service; import soot.SootClass; import soot.SootMethod; -import soot.tagkit.AnnotationTag; -import soot.tagkit.Tag; -import soot.tagkit.VisibilityAnnotationTag; +import tabby.common.bean.edge.Alias; +import tabby.common.bean.edge.Extend; import tabby.common.bean.edge.Has; +import tabby.common.bean.edge.Interfaces; import tabby.common.bean.ref.ClassReference; import tabby.common.bean.ref.MethodReference; +import tabby.common.utils.SemanticUtils; +import tabby.config.GlobalConfiguration; import tabby.core.container.DataContainer; import tabby.core.container.RulesContainer; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; /** @@ -27,178 +31,166 @@ public class ClassInfoCollector { @Autowired private DataContainer dataContainer; + @Autowired RulesContainer rulesContainer; + @Async("tabby-collector") public CompletableFuture collect(SootClass cls){ - return CompletableFuture.completedFuture(collect0(cls, dataContainer)); + // generate class reference + ClassReference classRef = collectClassInfo(cls, rulesContainer); + // generate method reference + collectMethodsInfo(classRef, cls, dataContainer, rulesContainer, false); + return CompletableFuture.completedFuture(classRef); } - /** - * 仅收集classRef，不保存到内存 - * @param cls - * @param dataContainer - * @return - */ - public static ClassReference collect0(SootClass cls, DataContainer dataContainer){ + public static ClassReference collectClassInfo(SootClass cls, RulesContainer rulesContainer){ ClassReference classRef = ClassReference.newInstance(cls); - Set relatedClassnames = getAllFatherNodes(cls); - classRef.setSerializable(relatedClassnames.contains("java.io.Serializable")); - classRef.setStrutsAction(relatedClassnames.contains("com.opensymphony.xwork2.ActionSupport") - || relatedClassnames.contains("com.opensymphony.xwork2.Action")); + return classRef; + } + + public static void collectMethodsInfo(ClassReference classRef, SootClass cls, + DataContainer dataContainer, RulesContainer rulesContainer, boolean newAdded){ + if(cls == null){ + cls = SemanticUtils.getSootClass(classRef.getName()); + } + // 提取类函数信息 - if(cls.getMethodCount() > 0){ - for (SootMethod method : cls.getMethods()) { - extractMethodInfo(method, classRef, relatedClassnames, dataContainer); + if(cls != null && cls.getMethodCount() > 0){ + List methods = new ArrayList<>(cls.getMethods()); + for (SootMethod method : methods) { + // check from db + MethodReference methodRef = dataContainer.getMethodRefBySignature(method.getSignature(), false); + if(methodRef != null) continue; + collectSingleMethodRef(classRef, method, newAdded, false, dataContainer, rulesContainer); } } - return classRef; } - - /** - * 提取函数基础信息，并保存到内存中 - * @param method - * @param ref - */ - public static void extractMethodInfo(SootMethod method, - ClassReference ref, - Set relatedClassnames, - DataContainer dataContainer - ){ - RulesContainer rulesContainer = dataContainer.getRulesContainer(); - String classname = ref.getName(); + public static MethodReference collectSingleMethodRef(ClassReference classRef, SootMethod method, boolean newAdded, boolean genAlias, + DataContainer dataContainer, RulesContainer rulesContainer){ + String classname = classRef.getName(); MethodReference methodRef = MethodReference.newInstance(classname, method); - rulesContainer.applyRule(classname, methodRef, relatedClassnames); - methodRef.setEndpoint(ref.isStrutsAction() || isEndpoint(method, relatedClassnames)); - methodRef.setNettyEndpoint(isNettyEndpoint(method, relatedClassnames)); - methodRef.setGetter(isGetter(method)); - methodRef.setSetter(isSetter(method)); - methodRef.setSerializable(relatedClassnames.contains("java.io.Serializable")); - methodRef.setAbstract(method.isAbstract()); - methodRef.setHasDefaultConstructor(ref.isHasDefaultConstructor()); - methodRef.setFromAbstractClass(ref.isAbstract()); - - Has has = Has.newInstance(ref, methodRef); - ref.getHasEdge().add(has); - dataContainer.store(has); + // apply rules + rulesContainer.applyRule(classname, methodRef); + // add to new added for next stage + if(GlobalConfiguration.IS_NEED_TO_DEAL_NEW_ADDED_METHOD && newAdded){ + dataContainer.getNewAddedMethodSigs().add(methodRef.getSignature()); + } + // generate has edge + generateHasEdge(classRef, methodRef, dataContainer, newAdded); dataContainer.store(methodRef); + if(genAlias){ + generateAliasEdge(methodRef, dataContainer); + } + return methodRef; } - - /** - * check method is an endpoint - * @param method - * @param relatedClassnames - * @return - */ - public static boolean isEndpoint(SootMethod method, Set relatedClassnames){ - // check jsp _jspService - if("_jspService".equals(method.getName())){ - return true; + public static ClassReference collectAndSave(Object clazz, boolean newAdded, + DataContainer dataContainer, RulesContainer rulesContainer){ + SootClass cls = null; + if(clazz instanceof SootClass){ + cls = (SootClass) clazz; + }else if(clazz instanceof String){ + cls = SemanticUtils.getSootClass((String) clazz); } - - // check from annotation - List tags = method.getTags(); - for (Tag tag : tags) { - if (tag instanceof VisibilityAnnotationTag) { - VisibilityAnnotationTag visibilityAnnotationTag = (VisibilityAnnotationTag) tag; - for (AnnotationTag annotationTag : visibilityAnnotationTag.getAnnotations()) { - String type = annotationTag.getType(); - if(type.endsWith("Mapping;") - || type.endsWith("javax/ws/rs/Path;") - || type.endsWith("javax/ws/rs/GET;") - || type.endsWith("javax/ws/rs/PUT;") - || type.endsWith("javax/ws/rs/DELETE;") - || type.endsWith("javax/ws/rs/POST;")){ - return true; - } - } + if(cls != null){ + if(rulesContainer == null){ + rulesContainer = GlobalConfiguration.rulesContainer; } + // generate class reference + ClassReference classRef = collectClassInfo(cls, rulesContainer); + // generate method reference + collectMethodsInfo(classRef, cls, dataContainer, rulesContainer, newAdded); + // save + dataContainer.store(classRef); + return classRef; } + return null; + } - // https://blog.csdn.net/melissa_heixiu/article/details/52472450 - List requestTypes = new ArrayList<>( - Arrays.asList("doGet","doPost","doPut","doDelete","doHead","doOptions","doTrace","service")); - // check from servlet - if((relatedClassnames.contains("javax.servlet.Servlet") - || relatedClassnames.contains("javax.servlet.http.HttpServlet") // 防止依赖缺失情况下的识别 - || relatedClassnames.contains("javax.servlet.GenericServlet")) - && requestTypes.contains(method.getName())){ - return true; - } - // not an endpoint - return false; + public static void collectRuntimeForSingleClazz(Object clazz, boolean newAdded, + DataContainer dataContainer, RulesContainer rulesContainer){ + ClassReference classReference = collectAndSave(clazz, newAdded, dataContainer, rulesContainer); + collectRelationInfo(classReference, newAdded, dataContainer, rulesContainer); } - public static boolean isNettyEndpoint(SootMethod method, Set relatedClassnames){ - String classname = method.getDeclaringClass().getName(); - if("io.netty.channel.ChannelInboundHandler".equals(classname) - || "io.netty.handler.codec.ByteToMessageDecoder".equals(classname) - ){ - return false; - } + public static void collectRelationInfo(ClassReference classRef, boolean newAdded, + DataContainer dataContainer, RulesContainer rulesContainer){ + if(classRef == null) return; + // 建立继承关系 + if(classRef.isHasSuperClass() && !"java.lang.Object".equals(classRef.getSuperClass())){ + ClassReference superClsRef = dataContainer.getClassRefByName(classRef.getSuperClass()); - String methodName = method.getName(); - // check from ChannelInboundHandler - List nettyReadMethods = Arrays.asList("channelRead", "channelRead0", "messageReceived"); - if(relatedClassnames.contains("io.netty.channel.ChannelInboundHandler") - && nettyReadMethods.contains(methodName)){ - return true; + if(superClsRef == null){ + superClsRef = collectAndSave(classRef.getSuperClass(), newAdded, dataContainer, rulesContainer); + } + + if(superClsRef != null){ + Extend extend = Extend.newInstance(classRef, superClsRef); + dataContainer.store(extend); + superClsRef.getChildClassnames().add(classRef.getName()); + } } - // check from io.netty.handler.codec.ByteToMessageDecoder - if(relatedClassnames.contains("io.netty.handler.codec.ByteToMessageDecoder") - && "decode".equals(methodName)){ - return true; + // 建立接口关系 + if(classRef.isHasInterfaces()){ + List infaces = classRef.getInterfaces(); + for(String inface:infaces){ + ClassReference infaceClsRef = dataContainer.getClassRefByName(inface); + if(infaceClsRef == null){// 正常情况不会进入这个阶段 + infaceClsRef = collectAndSave(inface, false, dataContainer, rulesContainer); + } + if(infaceClsRef != null){ + Interfaces interfaces = Interfaces.newInstance(classRef, infaceClsRef); + dataContainer.store(interfaces); + infaceClsRef.getChildClassnames().add(classRef.getName()); + } + } + } + // 建立函数别名关系 + List hasEdges = classRef.getHasEdge(); + if(hasEdges != null && !hasEdges.isEmpty()){ + for(Has has:hasEdges){ + generateAliasEdge(has.getMethodRef(), dataContainer); + } } - // not an endpoint - return false; + // finished + classRef.setInitialed(true); } - public static boolean isGetter(SootMethod method){ - String methodName = method.getName(); - String returnType = method.getReturnType().toString(); - boolean noParameter = method.getParameterCount() == 0; - boolean isPublic = method.isPublic(); + public static void generateHasEdge(ClassReference classRef, MethodReference methodRef, + DataContainer dataContainer, boolean newAdded){ + Has has = Has.newInstance(classRef, methodRef); + if(classRef.getHasEdge().contains(has)) return; - if(!noParameter || !isPublic) return false; - - if(methodName.startsWith("get") && methodName.length() > 3){ - return !"void".equals(returnType); - }else if(methodName.startsWith("is") && methodName.length() > 2){ - return "boolean".equals(returnType); + classRef.getHasEdge().add(has); + dataContainer.store(has); + if(newAdded){ + generateAliasEdge(methodRef, dataContainer); } - - return false; } - public static boolean isSetter(SootMethod method){ - String methodName = method.getName(); - String returnType = method.getReturnType().toString(); - boolean singleParameter = method.getParameterCount() == 1; - boolean isPublic = method.isPublic(); - - if(!isPublic || !singleParameter) return false; + public static void generateAliasEdge(MethodReference methodRef, DataContainer dataContainer){ + String methodName = methodRef.getName(); - if(methodName.startsWith("set") && methodName.length() > 3){ - return "void".equals(returnType); + if("".equals(methodName) + || "".equals(methodName)){ + return; } - return false; - } + SootMethod currentSootMethod = methodRef.getMethod(); + if(currentSootMethod == null) return; + SootClass cls = currentSootMethod.getDeclaringClass(); - public static Set getAllFatherNodes(SootClass cls){ - Set nodes = new HashSet<>(); - if(cls.hasSuperclass() && !cls.getSuperclass().getName().equals("java.lang.Object")){ - nodes.add(cls.getSuperclass().getName()); - nodes.addAll(getAllFatherNodes(cls.getSuperclass())); - } - if(cls.getInterfaceCount() > 0){ - cls.getInterfaces().forEach(intface -> { - nodes.add(intface.getName()); - nodes.addAll(getAllFatherNodes(intface)); - }); + Set refs = + dataContainer.getAliasMethodRefs(cls, currentSootMethod.getSubSignature()); + + if(refs != null && !refs.isEmpty()){ + for(MethodReference ref:refs){ + Alias alias = Alias.newInstance(ref, methodRef); + ref.getChildAliasEdges().add(alias); + dataContainer.store(alias); + } } - return nodes; } } \ No newline at end of file diff --git a/src/main/java/tabby/core/container/DataContainer.java b/src/main/java/tabby/core/container/DataContainer.java index 6112e76..7828875 100644 --- a/src/main/java/tabby/core/container/DataContainer.java +++ b/src/main/java/tabby/core/container/DataContainer.java @@ -8,11 +8,14 @@ import soot.SootClass; import soot.SootMethod; import soot.SootMethodRef; +import soot.jimple.InvokeExpr; +import tabby.analysis.data.Context; import tabby.common.bean.edge.*; import tabby.common.bean.ref.ClassReference; import tabby.common.bean.ref.MethodReference; import tabby.common.utils.SemanticUtils; -import tabby.core.scanner.ClassInfoScanner; +import tabby.config.GlobalConfiguration; +import tabby.core.collector.ClassInfoCollector; import tabby.dal.service.ClassRefService; import tabby.dal.service.MethodRefService; import tabby.dal.service.RelationshipsService; @@ -51,6 +54,12 @@ public class DataContainer { private Set savedAliasNodes = Collections.synchronizedSet(new HashSet<>()); private Set savedExtendNodes = Collections.synchronizedSet(new HashSet<>()); private Set savedInterfacesNodes = Collections.synchronizedSet(new HashSet<>()); + private Map runningMethods = Collections.synchronizedMap(new HashMap<>()); + private Set newAddedMethodSigs = Collections.synchronizedSet(new HashSet<>()); + private Set targets = Collections.synchronizedSet(new HashSet<>()); + private Set analyseTimeoutMethodSigs = Collections.synchronizedSet(new HashSet<>()); + + /** * check size and save nodes @@ -105,6 +114,10 @@ public void save(String type){ } } + public Context getRunningMethodContext(String signature){ + return runningMethods.get(signature); + } + /** * store nodes * 保存节点到内存 @@ -122,6 +135,7 @@ public void store(T ref) { }else if(ref instanceof MethodReference){ MethodReference methodRef = (MethodReference) ref; savedMethodRefs.put(methodRef.getSignature(), methodRef); + targets.add(methodRef.getSignature()); }else if(ref instanceof Has){ savedHasNodes.add((Has) ref); }else if(ref instanceof Call){ @@ -231,39 +245,74 @@ public MethodReference getMethodRefBySignature(String classname, String subSigna return null; } - /** - * 对于找不到的methodref - * 1. 新建classRef 如果不存在的话 - * 2. 从classRef找methodRef - * 3. 如果还是找不到则新建 - * @param sootMethodRef - * @return - */ - public MethodReference getOrAddMethodRef(SootMethodRef sootMethodRef, SootMethod method){ + public MethodReference getMethodRefBySignature(String signature, boolean isNeedFromDatabase){ + MethodReference ref = savedMethodRefs.getOrDefault(clean(signature), null); + if(ref != null) return ref; + if(isNeedFromDatabase){ + // find from h2 + ref = methodRefService.getMethodRefBySignature(clean(signature)); + if(ref != null){ + store(ref); + } + } + return ref; + } + + public MethodReference getOrAddMethodRef(SootMethod method){ // 递归查找父节点 + String signature = method.getSignature(); + MethodReference methodRef = getMethodRefBySignature(signature, true); + + if(methodRef == null){ + SootClass cls = method.getDeclaringClass(); + methodRef = addMethodRef(cls, method); + } + return methodRef; + } + + public MethodReference getOrAddMethodRef(InvokeExpr ie){ + SootMethod sootMethod = SemanticUtils.getMethod(ie); + if(sootMethod != null){ + return getOrAddMethodRef(sootMethod); + } + + SootMethodRef sootMethodRef = ie.getMethodRef(); + MethodReference methodRef = getMethodRefBySignature(sootMethodRef); if(methodRef == null){ - // 解决ClassInfoScanner阶段，函数信息收集不完全的问题 - SootClass cls = sootMethodRef.getDeclaringClass(); - ClassReference classRef = getClassRefByName(cls.getName()); - if(classRef == null){// 对于新建的情况，再查一遍 - classRef = ClassInfoScanner.collect0(cls.getName(), cls, this, 0); - methodRef = getMethodRefBySignature(sootMethodRef); - } + methodRef = addMethodRef(sootMethodRef); + } + return methodRef; + } - if(methodRef == null){ - methodRef = MethodReference.newInstance(classRef.getName(), method); - rulesContainer.applyRule(cls.getName(), methodRef, new HashSet<>()); - Has has = Has.newInstance(classRef, methodRef); - if(!classRef.getHasEdge().contains(has)){ - classRef.getHasEdge().add(has); - store(has); - ClassInfoScanner.makeAliasRelation(has, this); - } - store(methodRef); - } + public MethodReference addMethodRef(SootMethodRef sootMethodRef){ + // 解决ClassInfoScanner阶段，函数信息收集不完全的问题 + SootClass cls = sootMethodRef.getDeclaringClass(); + SootMethod method = SemanticUtils.getMethod(sootMethodRef); + return addMethodRef(cls, method); + } + + public MethodReference addMethodRef(SootClass cls, SootMethod method){ + // 解决ClassInfoScanner阶段，函数信息收集不完全的问题 + MethodReference methodRef = null; + ClassReference classRef = getClassRefByName(cls.getName()); + if(classRef == null){// 对于新建的情况，再查一遍 + ClassInfoCollector.collectRuntimeForSingleClazz(cls.getName(), true, this, null); + methodRef = getMethodRefBySignature(method.getSignature(), true); + }else if(method != null && + ("soot.dummy.InvokeDynamic".equals(cls.getName()) +// || "java.lang.invoke.VarHandle".equals(cls.getName()) +// || "java.lang.invoke.MethodHandle".equals(cls.getName()) + || cls.getName().contains("$lambda_") + || method.isNative() + )){ + // force add Dynamic method + methodRef = ClassInfoCollector.collectSingleMethodRef(classRef, method, false,false, this, GlobalConfiguration.rulesContainer); + }else if(method != null){ + methodRef = ClassInfoCollector.collectSingleMethodRef(classRef, method, true,true, this, GlobalConfiguration.rulesContainer); } + return methodRef; } diff --git a/src/main/java/tabby/core/container/RulesContainer.java b/src/main/java/tabby/core/container/RulesContainer.java index 5773db9..e1b150f 100644 --- a/src/main/java/tabby/core/container/RulesContainer.java +++ b/src/main/java/tabby/core/container/RulesContainer.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import tabby.common.bean.ref.MethodReference; +import tabby.common.utils.SemanticUtils; import tabby.config.GlobalConfiguration; import tabby.common.rule.TabbyRule; import tabby.common.utils.FileUtils; @@ -43,8 +44,9 @@ public TabbyRule.Rule getRule(String classname, String method){ return null; } - public void applyRule(String classname, MethodReference methodRef, Set relatedClassnames){ + public void applyRule(String classname, MethodReference methodRef){ TabbyRule.Rule rule = getRule(classname, methodRef.getName()); + Set relatedClassnames = SemanticUtils.getAllFatherNodes(classname); if (rule == null) { // 对于ignore类型，支持多级父类和接口的规则查找 for (String relatedClassname : relatedClassnames) { diff --git a/src/main/java/tabby/core/scanner/CallGraphScanner.java b/src/main/java/tabby/core/scanner/CallGraphScanner.java index 10f0f4e..e563c17 100644 --- a/src/main/java/tabby/core/scanner/CallGraphScanner.java +++ b/src/main/java/tabby/core/scanner/CallGraphScanner.java @@ -12,8 +12,7 @@ import tabby.dal.service.MethodRefService; import tabby.common.utils.TickTock; -import java.util.ArrayList; -import java.util.Collection; +import java.util.*; /** * 收集所有调用关系，这部分不做污点分析 @@ -40,22 +39,60 @@ public void run() { } public void collect() { + if(GlobalConfiguration.IS_FULL_CALL_GRAPH_CONSTRUCT){ + collectAllCallEdge(); + }else{ + List targets = new ArrayList<>(dataContainer.getTargets());; + log.info("Build call graph. START!"); + log.info("Method Timeout on {} min.", GlobalConfiguration.METHOD_TIMEOUT); + doCollectWithNewAddedMethods(targets); + log.info("Build call graph. DONE!"); + } + } + + public void doCollectWithNewAddedMethods(List targets){ + boolean flag = true; + while(!targets.isEmpty()){ + doCollect(targets, flag); + targets = new ArrayList<>(dataContainer.getNewAddedMethodSigs()); + dataContainer.setNewAddedMethodSigs(Collections.synchronizedSet(new HashSet<>())); + + if(targets.size() > 0){ + log.info("Analyse {} newAddedMethods", targets.size()); + flag = false; + } + } + } + + public void doCollect(List targets, boolean show){ + int total = targets.size(); + TickTock tt = new TickTock(total, show); + Collections.shuffle(targets); + for(String signature:targets){ + MethodReference ref = dataContainer.getMethodRefBySignature(signature, false); + if(ref != null){ + callGraphCollector.collect(ref, dataContainer, tt); + }else{ + tt.countDown(); + } + } + tt.awaitUntilCompleted(); + } + + public void collectAllCallEdge(){ + log.info("Build call graph. START!"); Collection targets = new ArrayList<>(dataContainer.getSavedMethodRefs().values()); - log.info("Build call graph. START!"); TickTock tickTock = new TickTock(targets.size(), true); for (MethodReference target : targets) { - if(GlobalConfiguration.IS_FULL_CALL_GRAPH_CONSTRUCT){ - callEdgeCollector.collect(target, dataContainer, tickTock); - }else{ - callGraphCollector.collect(target, dataContainer, tickTock); - } + callEdgeCollector.collect(target, dataContainer, tickTock); } - tickTock.await(); + tickTock.awaitUntilCompleted(); log.info("Build call graph. DONE!"); } public void save() { + if(GlobalConfiguration.GLOBAL_FORCE_STOP) return; log.info("Save remained data to graphdb. START!"); dataContainer.save("class"); dataContainer.save("method"); diff --git a/src/main/java/tabby/core/scanner/ClassInfoScanner.java b/src/main/java/tabby/core/scanner/ClassInfoScanner.java index 252cc0e..09f233e 100644 --- a/src/main/java/tabby/core/scanner/ClassInfoScanner.java +++ b/src/main/java/tabby/core/scanner/ClassInfoScanner.java @@ -4,26 +4,21 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import soot.*; -import tabby.common.bean.edge.Alias; -import tabby.common.bean.edge.Extend; -import tabby.common.bean.edge.Has; -import tabby.common.bean.edge.Interfaces; +import soot.ModulePathSourceLocator; +import soot.Scene; +import soot.SootClass; import tabby.common.bean.ref.ClassReference; -import tabby.common.bean.ref.MethodReference; -import tabby.common.utils.JavaVersionUtils; import tabby.common.utils.SemanticUtils; import tabby.config.GlobalConfiguration; import tabby.core.collector.ClassInfoCollector; import tabby.core.container.DataContainer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import static tabby.config.GlobalConfiguration.rulesContainer; + /** * 处理jdk相关类的信息抽取 * @author wh1t3P1g @@ -61,7 +56,8 @@ public Map> loadAndExtract(List classes = getTargetClasses(path, moduleClasses); + List classes = SemanticUtils.getTargetClasses(path, moduleClasses); + if(classes == null) continue; for (String cl : classes) { @@ -85,37 +81,25 @@ public Map> loadAndExtract(List getTargetClasses(String filepath, Map> moduleClasses){ - List classes = null; - Path path = Paths.get(filepath); - if(Files.notExists(path)) return null; - - if(JavaVersionUtils.isAtLeast(9) && moduleClasses != null){ - String filename = path.getFileName().toString(); - if(filename.endsWith(".jmod")){ - filename = filename.substring(0, filename.length() - 5); - } - classes = moduleClasses.get(filename); - } - - if(classes == null){ - classes = SourceLocator.v().getClassesUnder(filepath); - } - - return classes; - } - public void transform(Collection> futures){ + int count = 0; for(CompletableFuture future:futures){ try { ClassReference classRef = future.get(); + if(count%10000==0){ + log.info("Collected {} classes' information", count); + } + count++; + if(classRef == null) continue; dataContainer.store(classRef); } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); + log.error(e.getMessage()); +// e.printStackTrace(); // 异步获取出错 } } + log.info("Collected {} classes' information", futures.size()); } public void buildClassEdges(List classes){ @@ -129,118 +113,13 @@ public void buildClassEdges(List classes){ counter++; ClassReference clsRef = dataContainer.getClassRefByName(cls); if(clsRef == null) continue; - extractRelationships(clsRef, dataContainer, 0); + ClassInfoCollector.collectRelationInfo(clsRef, false, dataContainer, rulesContainer); } log.info("Build {}/{} classes.", counter, total); } - public static void extractRelationships(ClassReference clsRef, DataContainer dataContainer, int depth){ - // 建立继承关系 - if(clsRef.isHasSuperClass()){ - ClassReference superClsRef = dataContainer.getClassRefByName(clsRef.getSuperClass()); - if(superClsRef == null && depth < 10){ // 正常情况不会进入这个阶段 - superClsRef = collect0(clsRef.getSuperClass(), null, dataContainer, depth+1); - } - if(superClsRef != null && !"java.lang.Object".equals(superClsRef.getName())){ - Extend extend = Extend.newInstance(clsRef, superClsRef); - dataContainer.store(extend); - } - } - - // 建立接口关系 - if(clsRef.isHasInterfaces()){ - List infaces = clsRef.getInterfaces(); - for(String inface:infaces){ - ClassReference infaceClsRef = dataContainer.getClassRefByName(inface); - if(infaceClsRef == null && depth < 10){// 正常情况不会进入这个阶段 - infaceClsRef = collect0(inface, null, dataContainer, depth+1); - } - if(infaceClsRef != null){ - Interfaces interfaces = Interfaces.newInstance(clsRef, infaceClsRef); - dataContainer.store(interfaces); - } - } - } - // 建立函数别名关系 - makeAliasRelations(clsRef, dataContainer); - } - - /** - * 根据单个类进行类信息收集 - * @param classname 待收集的类名 - * @return 具体的类信息 - */ - public static ClassReference collect0(String classname, SootClass cls, - DataContainer dataContainer, int depth){ - ClassReference classRef = null; - try{ - if(cls == null){ - cls = SemanticUtils.getSootClass(classname); - } - }catch (Exception e){ - // class not found - } - - if(cls != null) { - if(cls.isPhantom()){ - classRef = ClassReference.newInstance(cls); - classRef.setPhantom(true); - }else{ - classRef = ClassInfoCollector.collect0(cls, dataContainer); - - extractRelationships(classRef, dataContainer, depth); - } - }else if(!classname.isEmpty()){ - classRef = ClassReference.newInstance(classname); - classRef.setPhantom(true); - } - - dataContainer.store(classRef); - return classRef; - } - - - - public static void makeAliasRelations(ClassReference ref, DataContainer dataContainer){ - if(ref == null)return; - // build alias relationship - if(ref.getHasEdge() == null) return; - - List hasEdges = ref.getHasEdge(); - for(Has has:hasEdges){ - makeAliasRelation(has, dataContainer); - } - - ref.setInitialed(true); - } - - public static void makeAliasRelation(Has has, DataContainer dataContainer){ - MethodReference currentMethodRef = has.getMethodRef(); - - if("".equals(currentMethodRef.getName()) - || "".equals(currentMethodRef.getName())){ - return; - } - - SootMethod currentSootMethod = currentMethodRef.getMethod(); - if(currentSootMethod == null) return; - - SootClass cls = currentSootMethod.getDeclaringClass(); - - Set refs = - dataContainer.getAliasMethodRefs(cls, currentSootMethod.getSubSignature()); - - if(refs != null && !refs.isEmpty()){ - for(MethodReference ref:refs){ - Alias alias = Alias.newInstance(ref, currentMethodRef); - ref.getChildAliasEdges().add(alias); -// currentMethodRef.setAliasEdge(alias); - dataContainer.store(alias); - } - } - } - public void save(){ + if(GlobalConfiguration.GLOBAL_FORCE_STOP) return; log.info("Start to save remained data to graphdb."); dataContainer.save("class"); dataContainer.save("has"); diff --git a/src/main/resources/logback-spring.xml b/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..c0035b5 --- /dev/null +++ b/src/main/resources/logback-spring.xml @@ -0,0 +1,96 @@ + + + + + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + INFO + ACCEPT + DENY + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + ERROR + ACCEPT + DENY + + + + + ${LOGS}/warn.log + + %msg%n + + + + + ${LOGS}/warn-%d{yyyy-MM-dd}.%i.log + + + 10MB + + + + + WARN + ACCEPT + DENY + + + + + ${LOGS}/debug.log + + %d{HH:mm:ss.SSS} [%-5level] - %msg%n + + + + + ${LOGS}/debug-%d{yyyy-MM-dd}.%i.log + + + 10MB + + + + + DEBUG + ACCEPT + DENY + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file