add tag function

rules/tags.json

@@ -0,0 +1,94 @@
+[
+  {
+    "name": "web-tags",
+    "type": "annotation",
+    "value": "web",
+    "annotations": [
+      "%Mapping", "javax.ws.rs.%", "javax.jws.%"
+    ],
+    "classes": [],
+    "methods": [],
+    "whitelist": []
+  },
+  {
+    "name": "jsp",
+    "type": "method",
+    "value": "web",
+    "annotations": [],
+    "classes": [],
+    "methods": ["_jspService"],
+    "whitelist": []
+  },
+  {
+    "name": "struts actions",
+    "type": "class",
+    "value": "web",
+    "annotations": [],
+    "classes": [
+      "com.opensymphony.xwork2.ActionSupport",
+      "com.opensymphony.xwork2.Action",
+      "org.apache.struts.actions.DispatchAction"
+    ],
+    "methods": [],
+    "whitelist": [
+      "com.opensymphony.xwork2.ActionSupport",
+      "com.opensymphony.xwork2.Action",
+      "org.apache.struts.actions.DispatchAction"
+    ]
+  },
+  {
+    "name": "jfinal controller",
+    "type": "class",
+    "value": "web",
+    "annotations": [],
+    "classes": [
+      "com.jfinal.core.Controller"
+    ],
+    "methods": [],
+    "whitelist": [
+      "com.jfinal.core.Controller",
+      "weaver.weixin.core.controller.BaseController"
+    ]
+  },
+  {
+    "name": "servlet",
+    "type": "class&method",
+    "value": "web",
+    "annotations": [],
+    "classes": ["javax.servlet.Servlet","javax.servlet.http.HttpServlet","javax.servlet.GenericServlet"],
+    "methods": ["doGet","doPost","doPut","doDelete","doHead","doOptions","doTrace","service"],
+    "whitelist": []
+  },
+  {
+    "name": "netty-handler",
+    "type": "class&method",
+    "value": "netty",
+    "annotations": [],
+    "classes": [
+      "io.netty.channel.ChannelInboundHandler",
+      "org.jboss.netty.channel.SimpleChannelUpstreamHandler"
+    ],
+    "methods": ["channelRead", "channelRead0", "messageReceived"],
+    "whitelist": [
+      "io.netty.channel.ChannelInboundHandler",
+      "org.jboss.netty.channel.SimpleChannelUpstreamHandler"
+    ]
+  },
+  {
+    "name": "netty-decoder",
+    "type": "class&method",
+    "value": "netty",
+    "annotations": [],
+    "classes": [
+      "io.netty.handler.codec.ByteToMessageDecoder",
+      "io.netty.handler.codec.MessageToMessageDecoder",
+      "org.jboss.netty.handler.codec.frame.FrameDecoder"
+    ],
+    "methods": ["decode"],
+    "whitelist": [
+      "io.netty.handler.codec.ByteToMessageDecoder",
+      "io.netty.handler.codec.MessageToMessageDecoder",
+      "org.jboss.netty.handler.codec.frame.FrameDecoder"
+    ]
+  }
+]

src/main/java/tabby/common/bean/ref/MethodReference.java

@@ -47,6 +47,9 @@ public class MethodReference {
     private int parameterSize;
     private String vul;
     private transient int callCounter = 0;
+    private String type;
+    private String urlPath;
+
 
     // 没有用上parameters 删除
 //    @Column(columnDefinition = "TEXT")
@@ -178,6 +181,12 @@ public void setMethod(SootMethod method){
         }
     }
 
+    public void setType(String type) {
+        this.type = type;
+        this.isEndpoint = "web".equals(type);
+        this.isNettyEndpoint = "netty".equals(type);
+    }
+
     public void addAction(String key, String value){
         actions.put(key, value);
     }

src/main/java/tabby/common/rule/TagRule.java

@@ -0,0 +1,45 @@
+package tabby.common.rule;
+
+import lombok.Data;
+
+import java.util.Set;
+
+/**
+ * @author wh1t3p1g
+ * @since 2023/3/15
+ */
+@Data
+public class TagRule {
+
+    private String name;
+    private String type;
+    private String value;
+    private Set<String> annotations;
+    private Set<String> classes;
+    private Set<String> methods; // sub_signature
+    private Set<String> whitelist; // class 用于排除某些特殊的class
+
+    public boolean isAnnotationType(){
+        return type.contains("annotation");
+    }
+
+    public boolean isClassType(){
+        return type.contains("class");
+    }
+
+    public boolean isMethodType(){
+        return type.contains("method");
+    }
+
+    public boolean isInWhitelist(String classname){
+        return whitelist != null && whitelist.contains(classname);
+    }
+
+    public void addClasses(Set<String> data){
+        classes.addAll(data);
+    }
+
+    public void addMethods(Set<String> data){
+        methods.addAll(data);
+    }
+}

src/main/java/tabby/common/utils/SemanticUtils.java

@@ -395,20 +395,40 @@ public static Set<String> getHttpUrlPaths(Map<String, Map<String, Set<String>>>
         return uris;
     }
 
-    public static Set<String> getMRPCUrlPaths(Map<String, Map<String, Set<String>>> annotations) {
-        Set<String> uris = new HashSet<>();
-        for(Map.Entry<String, Map<String,Set<String>>> entry:annotations.entrySet()){
-            String annotationName = entry.getKey();
-            if(annotationName == null || annotationName.isEmpty()) continue;
+    public static String getHttpUrlPathWithBaseURLPaths(Map<String, Map<String, Set<String>>> annotations, Set<String> baseUrlPaths) {
+        Set<String> urls = new HashSet<>();
 
-            if(annotationName.endsWith("OperationType")){
-                Set<String> values = entry.getValue().get("value");
-                if(values != null){
-                    uris.addAll(values);
+        if (baseUrlPaths == null || baseUrlPaths.isEmpty()) {
+            baseUrlPaths = new HashSet<>();
+            baseUrlPaths.add("");
+        }
+
+        Set<String> subUrlPaths = getHttpUrlPaths(annotations);
+
+        for(String base:baseUrlPaths){
+            for(String sub:subUrlPaths){
+                String urlPath = base;
+                if(urlPath.isEmpty()){
+                    urlPath = sub;
+                }else{
+                    urlPath += "/"+sub;
+                }
+                if (urlPath.endsWith("/")) {
+                    urlPath = urlPath.substring(0, urlPath.length() - 1);
+                }
+
+                urlPath = urlPath.replace("//", "/");
+                if(!urlPath.isEmpty()){
+                    urls.add(urlPath);
                 }
             }
         }
-        return uris;
+
+        if(urls.isEmpty()){
+            return "";
+        }else{
+            return String.join(",", urls);
+        }
     }
 
     public static boolean isInterface(Type type){

src/main/java/tabby/config/GlobalConfiguration.java

@@ -30,6 +30,7 @@ public class GlobalConfiguration {
     public static String RULES_PATH;
     public static String SINK_RULE_PATH;
     public static String SYSTEM_RULE_PATH;
+    public static String TAG_RULE_PATH;
     public static String IGNORE_PATH;
     public static String BASIC_CLASSES_PATH;
     public static String COMMON_JARS_PATH;
@@ -92,6 +93,7 @@ public static void init(){
             SINK_RULE_PATH = String.join(File.separator, RULES_PATH, "sinks.json");
             SYSTEM_RULE_PATH = String.join(File.separator, RULES_PATH, "system.json");
             IGNORE_PATH = String.join(File.separator, RULES_PATH, "ignores.json");
+            TAG_RULE_PATH = String.join(File.separator, RULES_PATH, "tags.json");
             BASIC_CLASSES_PATH = String.join(File.separator, RULES_PATH, "basicClasses.json");
             COMMON_JARS_PATH = String.join(File.separator, RULES_PATH, "commonJars.json");
             THREAD_POOL_SIZE = getProperty("tabby.build.thread.size", "max", props);

src/main/java/tabby/core/collector/ClassInfoCollector.java

@@ -72,6 +72,7 @@ public static MethodReference collectSingleMethodRef(ClassReference classRef, So
         MethodReference methodRef = MethodReference.newInstance(classname, method);
         // apply rules
         rulesContainer.applyRule(classname, methodRef);
+        rulesContainer.applyTagRule(classRef, methodRef);
         // add to new added for next stage
         if(GlobalConfiguration.IS_NEED_TO_DEAL_NEW_ADDED_METHOD && newAdded){
             dataContainer.getNewAddedMethodSigs().add(methodRef.getSignature());

src/main/java/tabby/core/container/RulesContainer.java

@@ -3,11 +3,13 @@
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
+import tabby.common.bean.ref.ClassReference;
 import tabby.common.bean.ref.MethodReference;
-import tabby.common.utils.SemanticUtils;
-import tabby.config.GlobalConfiguration;
 import tabby.common.rule.TabbyRule;
+import tabby.common.rule.TagRule;
 import tabby.common.utils.FileUtils;
+import tabby.common.utils.SemanticUtils;
+import tabby.config.GlobalConfiguration;
 
 import java.io.FileNotFoundException;
 import java.util.*;
@@ -26,10 +28,14 @@ public class RulesContainer {
     private List<String> excludedClasses; // 不进行分析的类
     private List<String> basicClasses; // 已经分析过的jar包
     private List<String> commonJars;
+    private TagRule[] tagRules;
+    private Map<String, TagRule> tagRuleMap = new HashMap<>();
+
 
     public RulesContainer() throws FileNotFoundException {
         load();
         loadIgnore();
+        loadTagRules();
         loadBasicClasses();
         loadCommonJars();
     }
@@ -88,6 +94,72 @@ public void applyRule(String classname, MethodReference methodRef){
         methodRef.setSource(isSource);
     }
 
+    public void applyTagRule(ClassReference clsRef, MethodReference methodRef){
+        // 根据tags规则处理
+        String classname = clsRef.getName();
+        for(TagRule tagRule:tagRules){
+            boolean flag = false;
+            if(tagRule.isInWhitelist(classname)){
+                break;
+            }
+
+            if(tagRule.isAnnotationType()){
+                Set<String> annotations = methodRef.getAnnotations().keySet();
+                for(String annotation:annotations){
+                    flag = check(annotation, tagRule.getAnnotations());
+                    if(flag){
+                        break;
+                    }
+                }
+                if(!flag) continue;
+            }
+
+            if(tagRule.isClassType()){
+                flag = check(classname, tagRule.getClasses());
+                if(!flag){
+                    Set<String> relatedClassnames = SemanticUtils.getAllFatherNodes(classname);
+                    for(String related:relatedClassnames){
+                        flag = check(related, tagRule.getClasses());
+                        if(flag) break;
+                    }
+                }
+                if(!flag) continue;
+            }
+
+            if(tagRule.isMethodType()){
+                flag = check(methodRef.getName(), tagRule.getMethods());
+                if(!flag) continue;
+            }
+
+            if(flag){ // annotation && class && method
+                methodRef.setType(tagRule.getValue()); // 可能不止命中一次规则
+
+                if(methodRef.isEndpoint()){
+                    Set<String> baseUrlPaths = SemanticUtils.getHttpUrlPaths(clsRef.getAnnotations());
+                    String urlPath = SemanticUtils.getHttpUrlPathWithBaseURLPaths(methodRef.getAnnotations(), baseUrlPaths);
+                    methodRef.setUrlPath(urlPath);
+                }
+            }
+        }
+    }
+
+    public boolean check(String data, Set<String> rules){
+        boolean flag = false;
+        for(String rule:rules){
+            if(rule.startsWith("%")){
+                String tmp = rule.substring(1);
+                flag = data.endsWith(tmp);
+            }else if(rule.endsWith("%")){
+                String tmp = rule.substring(0, rule.length()-1);
+                flag = data.startsWith(tmp);
+            }else{
+                flag = data.equals(rule);
+            }
+            if(flag) return true;
+        }
+        return false;
+    }
+
     public TabbyRule getRule(String classname){
         return rules.getOrDefault(classname, null);
     }
@@ -130,6 +202,19 @@ private void load() throws FileNotFoundException {
         }
         log.info("load "+ rules.size() +" rules success!");
     }
+
+    private void loadTagRules(){
+        tagRules = (TagRule[]) FileUtils.getJsonContent(GlobalConfiguration.TAG_RULE_PATH, TagRule[].class);
+        if(tagRules == null){
+            tagRules = new TagRule[0];
+        }
+
+        for(TagRule tagRule:tagRules){
+            tagRuleMap.put(tagRule.getName(), tagRule);
+        }
+    }
+
+
     @SuppressWarnings({"unchecked"})
     private void loadIgnore(){
         ignored = (List<String>) FileUtils.getJsonContent(GlobalConfiguration.IGNORE_PATH, List.class);