Version: 2.6 Update

README.md

@@ -25,14 +25,16 @@
 
 ### 规则释义
 
-HaE目前的规则一共有6个字段，分别是规则名称、规则正则、规则作用域、正则引擎、规则匹配颜色、规则敏感性。
+HaE目前的规则一共有8个字段，分别是规则名称、规则正则、规则作用域、正则引擎、规则匹配颜色、规则敏感性。
 
 详细的含义如下所示：
 
 | 字段      | 含义                                                                                                                                                                                                   |
 |-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | Name      | 规则名称，主要用于简短概括当前规则的作用。                                                                                                                                                               |
-| Regex     | 规则正则，主要用于填写正则表达式。在HaE中所需提取匹配的内容需要用`(`、`)`将正则表达式进行包裹。                                                                                                            |
+| F-Regex     | 规则正则，主要用于填写正则表达式。在HaE中所需提取匹配的内容需要用`(`、`)`将正则表达式进行包裹。|
+| S-Regex     | 规则正则，作用及使用同F-Regex。S-Regex为二次正则，可以用于对F-Regex匹配的数据结果进行二次的匹配提取，如不需要的情况下可以留空。|
+| Format     | 格式化输出，在NFA引擎的正则表达式中，我们可以通过`{0}`、`{1}`、`{2}`…的方式进行取分组格式化输出。默认情况下使用`{0}`即可。          |
 | Scope     | 规则作用域，主要用于表示当前规则作用于HTTP报文的哪个部分。                                                                                                                                               |
 | Engine    | 正则引擎，主要用于表示当前规则的正则表达式所使用的引擎。**DFA引擎**：对于文本串里的每一个字符只需扫描一次，速度快、特性少；**NFA引擎**：要翻来覆去标注字符、取消标注字符，速度慢，但是特性（如:分组、替换、分割）丰富。 |
 | Color     | 规则匹配颜色，主要用于表示当前规则匹配到对应HTTP报文时所需标记的高亮颜色。在HaE中具备颜色升级算法，当出现相同颜色时会自动向上升级一个颜色进行标记。                                                                                                                               |

src/main/java/burp/BurpExtender.java

@@ -36,7 +36,7 @@ public void registerExtenderCallbacks(final IBurpExtenderCallbacks callbacks)
 
         new ConfigLoader();
 
-        String version = "2.5.11";
+        String version = "2.6";
         callbacks.setExtensionName(String.format("HaE (%s) - Highlighter and Extractor", version));
 
         // 定义输出

src/main/java/burp/config/ConfigLoader.java

@@ -112,7 +112,7 @@ public static String getExcludeSuffix(){
     public static Map<String, Object[][]> getRules() {
         Map<String, Object> rulesMap = YamlTool.loadYaml(getRulesFilePath());
         Map<String, Object[][]> resRule = new HashMap<>();
-        String[] fieldKeys = {"loaded", "name", "regex", "color", "scope", "engine", "sensitive"};
+        String[] fieldKeys = {"loaded", "name", "f_regex", "s_regex", "format", "color", "scope", "engine", "sensitive"};
 
         Object rulesObj = rulesMap.get("rules");
         if (rulesObj instanceof List) {

src/main/java/burp/core/processor/DataProcessingUnit.java

@@ -12,6 +12,7 @@
 import dk.brics.automaton.RunAutomaton;
 import java.nio.charset.StandardCharsets;
 import java.security.NoSuchAlgorithmException;
+import java.text.MessageFormat;
 import java.util.*;
 
 import java.util.concurrent.ConcurrentHashMap;
@@ -66,13 +67,16 @@ public Map<String, Map<String, Object>> matchContentByRegex(byte[] content, Stri
                     List<String> result = new ArrayList<>();
                     Map<String, Object> tmpMap = new HashMap<>();
 
-                    String name = objects[1].toString();
                     boolean loaded = (Boolean) objects[0];
-                    String regex = objects[2].toString();
-                    String color = objects[3].toString();
-                    String scope = objects[4].toString();
-                    String engine = objects[5].toString();
-                    boolean sensitive = (Boolean) objects[6];
+                    String name = objects[1].toString();
+                    String f_regex = objects[2].toString();
+                    String s_regex = objects[3].toString();
+                    String format = objects[4].toString();
+                    String color = objects[5].toString();
+                    String scope = objects[6].toString();
+                    String engine = objects[7].toString();
+                    boolean sensitive = (Boolean) objects[8];
+
                     // 判断规则是否开启与作用域
                     if (loaded && (scope.contains(scopeString) || scope.contains("any"))) {
                         switch (scope) {
@@ -96,34 +100,9 @@ public Map<String, Map<String, Object>> matchContentByRegex(byte[] content, Stri
                         }
 
                         try {
-                            if ("nfa".equals(engine)) {
-                                Pattern pattern;
-                                // 判断规则是否大小写敏感
-                                if (sensitive) {
-                                    pattern = new Pattern(regex);
-                                } else {
-                                    pattern = new Pattern(regex, Pattern.IGNORE_CASE);
-                                }
-
-                                Matcher matcher = pattern.matcher(matchContent);
-                                while (matcher.find()) {
-                                    // 添加匹配数据至list
-                                    // 强制用户使用()包裹正则
-                                    result.add(matcher.group(1));
-                                }
-                            } else {
-                                RegExp regexp = new RegExp(regex);
-                                Automaton auto = regexp.toAutomaton();
-                                RunAutomaton runAuto = new RunAutomaton(auto, true);
-                                AutomatonMatcher autoMatcher = runAuto.newMatcher(matchContent);
-                                while (autoMatcher.find()) {
-                                    // 添加匹配数据至list
-                                    // 强制用户使用()包裹正则
-                                    result.add(autoMatcher.group());
-                                }
-                            }
+                            result.addAll(matchByRegex(f_regex, s_regex, matchContent, format, engine, sensitive));
                         } catch (Exception e) {
-                            BurpExtender.stdout.println(String.format("[x] Error Info:\nName: %s\nRegex: %s", name, regex));
+                            BurpExtender.stdout.println(String.format("[x] Error Info:\nName: %s\nRegex: %s", name, f_regex));
                             e.printStackTrace();
                             continue;
                         }
@@ -190,6 +169,122 @@ public Map<String, Map<String, Object>> matchContentByRegex(byte[] content, Stri
             GlobalCachePool.addToCache(messageIndex, finalMap);
             return finalMap;
         }
+    }
+
+    private List<String> matchByRegex(String f_regex, String s_regex, String content, String format, String engine, boolean sensitive) {
+        List<String> retList = new ArrayList<>();
+        if ("nfa".equals(engine)) {
+            Matcher matcher = createPatternMatcher(f_regex, content, sensitive);
+            retList.addAll(extractMatches(s_regex, format, sensitive, matcher));
+        } else {
+            String newContent = content;
+            String newFirstRegex = f_regex;
+            if (!sensitive) {
+                newContent = content.toLowerCase();
+                newFirstRegex = f_regex.toLowerCase();
+            }
+            AutomatonMatcher autoMatcher = createAutomatonMatcher(newFirstRegex, newContent);
+            retList.addAll(extractMatches(s_regex, format, autoMatcher, content));
+        }
+        return retList;
+    }
+
+    private List<String> extractMatches(String s_regex, String format, boolean sensitive, Matcher matcher) {
+        List<String> matches = new ArrayList<>();
+        if (s_regex.isEmpty()) {
+            matches.addAll(getFormatString(matcher, format));
+        } else {
+            while (matcher.find()) {
+                matcher = createPatternMatcher(s_regex, matcher.group(1), sensitive);
+                matches.addAll(getFormatString(matcher, format));
+            }
+        }
+        return matches;
+    }
+
+    private List<String> extractMatches(String s_regex, String format, AutomatonMatcher autoMatcher, String content) {
+        List<String> matches = new ArrayList<>();
+        if (s_regex.isEmpty()) {
+            matches.addAll(getFormatString(autoMatcher, format, content));
+        } else {
+            while (autoMatcher.find()) {
+                autoMatcher = createAutomatonMatcher(s_regex, getSubString(content, autoMatcher.group()));
+                matches.addAll(getFormatString(autoMatcher, format, content));
+            }
+        }
+        return matches;
+    }
+
+    public List<String> getFormatString(Matcher matcher, String format) {
+        List<Integer> indexList = parseIndexesFromString(format);
+        List<String> stringList = new ArrayList<>();
+
+        while (matcher.find()) {
+            Object[] params = indexList.stream().map(i -> {
+                if (matcher.group(i+1) != null) {
+                    return matcher.group(i+1);
+                }
+                return "";
+            }).toArray();
+            stringList.add(MessageFormat.format(reorderIndex(format), params));
+        }
+
+        return stringList;
+    }
+
+    public List<String> getFormatString(AutomatonMatcher matcher, String format, String content) {
+        List<Integer> indexList = parseIndexesFromString(format);
+        List<String> stringList = new ArrayList<>();
+
+        while (matcher.find()) {
+            Object[] params = indexList.stream().map(i -> getSubString(content, matcher.group(i))).toArray();
+            stringList.add(MessageFormat.format(reorderIndex(format), params));
+        }
+
+        return stringList;
+    }
+
+    private Matcher createPatternMatcher(String regex, String content, boolean sensitive) {
+        Pattern pattern = (sensitive) ? new Pattern(regex) : new Pattern(regex, Pattern.IGNORE_CASE);
+        return pattern.matcher(content);
+    }
 
+    private AutomatonMatcher createAutomatonMatcher(String regex, String content) {
+        RegExp regexp = new RegExp(regex);
+        Automaton auto = regexp.toAutomaton();
+        RunAutomaton runAuto = new RunAutomaton(auto, true);
+        return runAuto.newMatcher(content);
     }
-}
+
+    private LinkedList<Integer> parseIndexesFromString(String input) {
+        LinkedList<Integer> indexes = new LinkedList<>();
+        Pattern pattern = new Pattern("\\{(\\d+)}");
+        Matcher matcher = pattern.matcher(input);
+
+        while (matcher.find()) {
+            indexes.add(Integer.valueOf(matcher.group(1)));
+        }
+
+        return indexes;
+    }
+
+    private String getSubString(String content, String s) {
+        int startIndex = content.toLowerCase().indexOf(s);
+        int endIndex = startIndex + s.length();
+        return content.substring(startIndex, endIndex);
+    }
+
+    private String reorderIndex(String format) {
+        Pattern pattern = new Pattern("\\{(\\d+)}");
+        Matcher matcher = pattern.matcher(format);
+        int count = 0;
+        while (matcher.find()) {
+            String newStr = String.format("{%s}", count);
+            String matchStr = matcher.group(0);
+            format = format.replace(matchStr, newStr);
+            count++;
+        }
+        return format;
+    }
+}
+

src/main/java/burp/rule/RuleProcessor.java

@@ -33,7 +33,9 @@ public void rulesFormatAndSave() {
                             (String) objects[3],
                             (String) objects[4],
                             (String) objects[5],
-                            (boolean) objects[6]))
+                            (String) objects[6],
+                            (String) objects[7],
+                            (boolean) objects[8]))
                     .collect(Collectors.toList());
             ruleGroupList.add(new RuleGroup(k, ruleList));
         });
@@ -80,17 +82,20 @@ public void deleteRuleGroup(String Rules) {
         ConfigEntry.globalRules.remove(Rules);
         this.rulesFormatAndSave();
     }
+
     public String newRule() {
         int i = 0;
         String name = "New ";
         Object[][] data = new Object[][] {
                 {
-                    false, "New Name", "(New Regex)", "gray", "any", "nfa", false
+                    false, "New Name", "(First Regex)", "(Second Regex)", "{0}", "gray", "any", "nfa", false
                 }
         };
+
         while (ConfigEntry.globalRules.containsKey(name + i)) {
             i++;
         }
+
         ConfigEntry.globalRules.put(name + i, data);
         this.rulesFormatAndSave();
         return name + i;

src/main/java/burp/rule/model/Rule.java

@@ -10,11 +10,13 @@
 public class Rule {
     private Map<String, Object> fields;
 
-    public Rule(boolean loaded, String name, String regex, String color, String scope, String engine, boolean sensitive) {
+    public Rule(boolean loaded, String name, String f_regex, String s_regex, String format, String color, String scope, String engine, boolean sensitive) {
         fields = new LinkedHashMap<>();
         fields.put("name", name);
         fields.put("loaded", loaded);
-        fields.put("regex", regex);
+        fields.put("f_regex", f_regex);
+        fields.put("s_regex", s_regex);
+        fields.put("format", format);
         fields.put("color", color);
         fields.put("scope", scope);
         fields.put("engine", engine);

src/main/java/burp/ui/MainUI.java

@@ -294,7 +294,7 @@ protected TabTitleEditListener(JTabbedPane tabbedPane) {
     }
 
     public void newTab(){
-        Object[][] data = new Object[][]{{false, "New Name", "(New Regex)", "gray", "any", "nfa", false}};
+        Object[][] data = new Object[][]{{false, "New Name", "(New Regex)", "", "{0}", "gray", "any", "nfa", false}};
         insertTab(ruleEditTabbedPane, ruleProcessor.newRule(),data);
     }
 

src/main/java/burp/ui/board/Databoard.java

@@ -109,9 +109,28 @@ private void initComponents() {
                 GridBagConstraints.CENTER, GridBagConstraints.BOTH,
                 new Insets(8, 0, 5, 5), 0, 0));
 
+        splitPane.addComponentListener(new ComponentAdapter() {
+            @Override
+            public void componentResized(ComponentEvent e) {
+                resizePanel();
+            }
+        });
+
         setAutoMatch();
     }
 
+    private void resizePanel() {
+        splitPane.setDividerLocation(0.4);
+        TableColumnModel columnModel = table.getColumnModel();
+        int totalWidth = (int) (getWidth() * 0.6);
+        columnModel.getColumn(0).setPreferredWidth((int) (totalWidth * 0.1));
+        columnModel.getColumn(1).setPreferredWidth((int) (totalWidth * 0.3));
+        columnModel.getColumn(2).setPreferredWidth((int) (totalWidth * 0.3));
+        columnModel.getColumn(3).setPreferredWidth((int) (totalWidth * 0.1));
+        columnModel.getColumn(4).setPreferredWidth((int) (totalWidth * 0.1));
+        columnModel.getColumn(5).setPreferredWidth((int) (totalWidth * 0.1));
+    }
+
     private static List<String> getHostByList() {
         return new ArrayList<>(ConfigEntry.globalDataMap.keySet());
     }
@@ -236,6 +255,7 @@ private void populateTabbedPaneByHost(String selectedHost) {
             dataTabbedPane.removeAll();
             dataTabbedPane.setPreferredSize(new Dimension(500,0));
             dataTabbedPane.setTabLayoutPolicy(JTabbedPane.SCROLL_TAB_LAYOUT);
+            dataTabbedPane.removeChangeListener(changeListenerInstance);
             splitPane.setLeftComponent(dataTabbedPane);
 
             if (selectedHost.contains("*")) {
@@ -263,10 +283,12 @@ private void populateTabbedPaneByHost(String selectedHost) {
             }
 
             if (selectedHost.equals("**")) {
+                if (currentWorker != null && !currentWorker.isDone()) {
+                    currentWorker.cancel(true);
+                }
                 for (ConcurrentHashMap.Entry<String, Map<String, List<String>>> entry : dataMap.entrySet()) {
                     JTabbedPane newTabbedPane = new JTabbedPane();
                     newTabbedPane.setTabLayoutPolicy(JTabbedPane.SCROLL_TAB_LAYOUT);
-
                     for (Map.Entry<String, List<String>> entrySet : entry.getValue().entrySet()) {
                         currentWorker = new SwingWorker<Object, Void>() {
                             @Override
@@ -300,8 +322,6 @@ protected void done() {
 
                 dataTabbedPane.addChangeListener(changeListenerInstance);
             } else {
-                dataTabbedPane.removeChangeListener(changeListenerInstance);
-
                 for (Map.Entry<String, List<String>> entry : selectedDataMap.entrySet()) {
                     String tabTitle = String.format("%s (%s)", entry.getKey(), entry.getValue().size());
                     DatatablePanel datatablePanel = new DatatablePanel(entry.getKey(), entry.getValue());
@@ -315,21 +335,7 @@ protected void done() {
             this.splitPane.setRightComponent(messageSplitPane);
             table = this.messagePanel.getTable();
 
-            this.splitPane.addComponentListener(new ComponentAdapter() {
-                @Override
-                public void componentResized(ComponentEvent e) {
-                    splitPane.setDividerLocation(0.4);
-                    TableColumnModel columnModel = table.getColumnModel();
-                    int totalWidth = (int) (getWidth() * 0.6);
-                    columnModel.getColumn(0).setPreferredWidth((int) (totalWidth * 0.1));
-                    columnModel.getColumn(1).setPreferredWidth((int) (totalWidth * 0.3));
-                    columnModel.getColumn(2).setPreferredWidth((int) (totalWidth * 0.3));
-                    columnModel.getColumn(3).setPreferredWidth((int) (totalWidth * 0.1));
-                    columnModel.getColumn(4).setPreferredWidth((int) (totalWidth * 0.1));
-                    columnModel.getColumn(5).setPreferredWidth((int) (totalWidth * 0.1));
-                }
-            });
-
+            resizePanel();
             splitPane.setVisible(true);
 
             applyHostFilter(selectedHost);

src/main/java/burp/ui/board/DatatablePanel.java

@@ -66,7 +66,9 @@ public int compare(Integer s1, Integer s2) {
         idColumn.setMaxWidth(50);
 
         for (String item : list) {
-            addRowToTable(model, new Object[]{item});
+            if (!item.isEmpty()) {
+                addRowToTable(model, new Object[]{item});
+            }
         }
 
         String defaultText = "Search";