Merge branch 'master' into fix/ida90

.justfile

@@ -16,5 +16,10 @@
 @deptry:
     pre-commit run deptry --hook-stage manual --all-files
 
-lint: isort black ruff flake8 mypy deptry
-
+@lint:
+    -just isort
+    -just black
+    -just ruff
+    -just flake8
+    -just mypy
+    -just deptry

CHANGELOG.md

@@ -10,8 +10,9 @@ Unlock powerful malware analysis with capa's new [VMRay sandbox](https://www.vmr
 - add rules website https://mandiant.github.io/capa/rules @DeeyaSingh #2310
 - add .justfile @williballenthin #2325
 - dynamic: add support for VMRay dynamic sandbox traces #2208 @mike-hunhoff @r-sm2024 @mr-tz
+- cli: use modern terminal features to hyperlink to the rules website #2337 @williballenthin
 - update IDAPython to IDA Pro 9.0 @mr-tz
- 
+
 ### Breaking Changes
 
 ### New Rules (0)

capa/render/default.py

@@ -6,18 +6,43 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import io
 import collections
+import urllib.parse
 
-import tabulate
+import rich
+import rich.table
+import rich.console
+from rich.console import Console
 
 import capa.render.utils as rutils
 import capa.render.result_document as rd
 import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
-from capa.render.utils import StringIO
 
-tabulate.PRESERVE_WHITESPACE = True
+
+def bold_markup(s) -> str:
+    """
+    Generate Rich markup in a bold style.
+
+    The resulting string should be passed to a Rich renderable
+    and/or printed via Rich or the markup will be visible to the user.
+    """
+    return f"[cyan]{s}[/cyan]"
+
+
+def link_markup(s: str, href: str) -> str:
+    """
+    Generate Rich markup for a clickable hyperlink.
+    This works in many modern terminals.
+    When it doesn't work, the fallback is just to show the link name (s),
+     as if it was not a link.
+
+    The resulting string should be passed to a Rich renderable
+    and/or printed via Rich or the markup will be visible to the user.
+    """
+    return f"[link={href}]{s}[/link]"
 
 
 def width(s: str, character_count: int) -> str:
@@ -28,20 +53,31 @@ def width(s: str, character_count: int) -> str:
         return s
 
 
-def render_meta(doc: rd.ResultDocument, ostream: StringIO):
+def render_sample_link(hash: str) -> str:
+    url = "https://www.virustotal.com/gui/file/" + hash
+    return link_markup(hash, url)
+
+
+def render_meta(doc: rd.ResultDocument, console: Console):
     rows = [
-        (width("md5", 22), width(doc.meta.sample.md5, 82)),
-        ("sha1", doc.meta.sample.sha1),
-        ("sha256", doc.meta.sample.sha256),
+        ("md5", render_sample_link(doc.meta.sample.md5)),
+        ("sha1", render_sample_link(doc.meta.sample.sha1)),
+        ("sha256", render_sample_link(doc.meta.sample.sha256)),
         ("analysis", doc.meta.flavor.value),
         ("os", doc.meta.analysis.os),
         ("format", doc.meta.analysis.format),
         ("arch", doc.meta.analysis.arch),
         ("path", doc.meta.sample.path),
     ]
 
-    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
-    ostream.write("\n")
+    table = rich.table.Table(show_header=False, min_width=100)
+    table.add_column()
+    table.add_column()
+
+    for row in rows:
+        table.add_row(*row)
+
+    console.print(table)
 
 
 def find_subrule_matches(doc: rd.ResultDocument):
@@ -71,7 +107,12 @@ def rec(match: rd.Match):
     return matches
 
 
-def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
+def render_rule_name(name: str) -> str:
+    url = f"https://mandiant.github.io/capa/rules/{urllib.parse.quote(name)}/"
+    return bold_markup(link_markup(name, url))
+
+
+def render_capabilities(doc: rd.ResultDocument, console: Console):
     """
     example::
 
@@ -95,25 +136,30 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
 
         count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule.meta.name)
+            capability = render_rule_name(rule.meta.name)
         else:
-            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
+            capability = render_rule_name(rule.meta.name) + f" ({count} matches)"
         rows.append((capability, rule.meta.namespace))
 
     if rows:
-        ostream.write(
-            tabulate.tabulate(
-                rows,
-                headers=[width("Capability", 50), width("Namespace", 50)],
-                tablefmt="mixed_outline",
-            )
-        )
-        ostream.write("\n")
+        table = rich.table.Table(min_width=100)
+        table.add_column(width("Capability", 20))
+        table.add_column("Namespace")
+
+        for row in rows:
+            table.add_row(*row)
+
+        console.print(table)
     else:
-        ostream.writeln(rutils.bold("no capabilities found"))
+        console.print(bold_markup("no capabilities found"))
+
+
+def render_attack_link(id: str) -> str:
+    url = f"https://attack.mitre.org/techniques/{id.replace('.', '/')}/"
+    return rf"\[{link_markup(id, url)}]"
 
 
-def render_attack(doc: rd.ResultDocument, ostream: StringIO):
+def render_attack(doc: rd.ResultDocument, console: Console):
     """
     example::
 
@@ -132,35 +178,36 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     tactics = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
         for attack in rule.meta.attack:
-            tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id))
+            tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id.strip("[").strip("]")))
 
     rows = []
     for tactic, techniques in sorted(tactics.items()):
         inner_rows = []
         for technique, subtechnique, id in sorted(techniques):
             if not subtechnique:
-                inner_rows.append(f"{rutils.bold(technique)} {id}")
+                # example: File and Directory Discovery [T1083]
+                inner_rows.append(f"{bold_markup(technique)} {render_attack_link(id)}")
             else:
-                inner_rows.append(f"{rutils.bold(technique)}::{subtechnique} {id}")
-        rows.append(
-            (
-                rutils.bold(tactic.upper()),
-                "\n".join(inner_rows),
-            )
-        )
+                # example: Code Discovery::Enumerate PE Sections [T1084.001]
+                inner_rows.append(f"{bold_markup(technique)}::{subtechnique} {render_attack_link(id)}")
+
+        tactic = bold_markup(tactic.upper())
+        technique = "\n".join(inner_rows)
+
+        rows.append((tactic, technique))
 
     if rows:
-        ostream.write(
-            tabulate.tabulate(
-                rows,
-                headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)],
-                tablefmt="mixed_grid",
-            )
-        )
-        ostream.write("\n")
+        table = rich.table.Table(min_width=100)
+        table.add_column(width("ATT&CK Tactic", 20))
+        table.add_column("ATT&CK Technique")
+
+        for row in rows:
+            table.add_row(*row)
 
+        console.print(table)
 
-def render_maec(doc: rd.ResultDocument, ostream: StringIO):
+
+def render_maec(doc: rd.ResultDocument, console: Console):
     """
     example::
 
@@ -193,20 +240,37 @@ def render_maec(doc: rd.ResultDocument, ostream: StringIO):
     for category in sorted(maec_categories):
         values = maec_table.get(category, set())
         if values:
-            rows.append((rutils.bold(category.replace("_", "-")), "\n".join(sorted(values))))
+            rows.append((bold_markup(category.replace("_", "-")), "\n".join(sorted(values))))
 
     if rows:
-        ostream.write(
-            tabulate.tabulate(
-                rows,
-                headers=[width("MAEC Category", 25), width("MAEC Value", 75)],
-                tablefmt="mixed_grid",
-            )
-        )
-        ostream.write("\n")
+        table = rich.table.Table(min_width=100)
+        table.add_column(width("MAEC Category", 20))
+        table.add_column("MAEC Value")
+
+        for row in rows:
+            table.add_row(*row)
+
+        console.print(table)
+
+
+def render_mbc_link(id: str, objective: str, behavior: str) -> str:
+    if id[0] in {"B", "T", "E", "F"}:
+        # behavior
+        base_url = "https://github.com/MBCProject/mbc-markdown/blob/main"
+    elif id[0] == "C":
+        # micro-behavior
+        base_url = "https://github.com/MBCProject/mbc-markdown/blob/main/micro-behaviors"
+    else:
+        raise ValueError("unexpected MBC prefix")
 
+    objective_fragment = objective.lower().replace(" ", "-")
+    behavior_fragment = behavior.lower().replace(" ", "-")
 
-def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
+    url = f"{base_url}/{objective_fragment}/{behavior_fragment}.md"
+    return rf"\[{link_markup(id, url)}]"
+
+
+def render_mbc(doc: rd.ResultDocument, console: Console):
     """
     example::
 
@@ -223,48 +287,48 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
     objectives = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
         for mbc in rule.meta.mbc:
-            objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id))
+            objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id.strip("[").strip("]")))
 
     rows = []
     for objective, behaviors in sorted(objectives.items()):
         inner_rows = []
-        for behavior, method, id in sorted(behaviors):
-            if not method:
-                inner_rows.append(f"{rutils.bold(behavior)} [{id}]")
+        for technique, subtechnique, id in sorted(behaviors):
+            if not subtechnique:
+                # example: File and Directory Discovery [T1083]
+                inner_rows.append(f"{bold_markup(technique)} {render_mbc_link(id, objective, technique)}")
             else:
-                inner_rows.append(f"{rutils.bold(behavior)}::{method} [{id}]")
-        rows.append(
-            (
-                rutils.bold(objective.upper()),
-                "\n".join(inner_rows),
-            )
-        )
+                # example: Code Discovery::Enumerate PE Sections [T1084.001]
+                inner_rows.append(
+                    f"{bold_markup(technique)}::{subtechnique} {render_mbc_link(id, objective, technique)}"
+                )
+
+        objective = bold_markup(objective.upper())
+        technique = "\n".join(inner_rows)
+
+        rows.append((objective, technique))
 
     if rows:
-        ostream.write(
-            tabulate.tabulate(
-                rows,
-                headers=[width("MBC Objective", 25), width("MBC Behavior", 75)],
-                tablefmt="mixed_grid",
-            )
-        )
-        ostream.write("\n")
+        table = rich.table.Table(min_width=100)
+        table.add_column(width("MBC Objective", 20))
+        table.add_column("MBC Behavior")
+
+        for row in rows:
+            table.add_row(*row)
+
+        console.print(table)
 
 
 def render_default(doc: rd.ResultDocument):
-    ostream = rutils.StringIO()
-
-    render_meta(doc, ostream)
-    ostream.write("\n")
-    render_attack(doc, ostream)
-    ostream.write("\n")
-    render_maec(doc, ostream)
-    ostream.write("\n")
-    render_mbc(doc, ostream)
-    ostream.write("\n")
-    render_capabilities(doc, ostream)
-
-    return ostream.getvalue()
+    f = io.StringIO()
+    console = rich.console.Console()
+
+    render_meta(doc, console)
+    render_attack(doc, console)
+    render_maec(doc, console)
+    render_mbc(doc, console)
+    render_capabilities(doc, console)
+
+    return f.getvalue()
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:

pyproject.toml

@@ -136,7 +136,7 @@ dev = [
     "flake8-simplify==0.21.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.6.2",
+    "ruff==0.6.4",
     "black==24.8.0",
     "isort==5.13.2",
     "mypy==1.11.2",
@@ -148,7 +148,7 @@ dev = [
     "types-PyYAML==6.0.8",
     "types-tabulate==0.9.0.20240106",
     "types-termcolor==1.1.4",
-    "types-psutil==6.0.0.20240621",
+    "types-psutil==6.0.0.20240901",
     "types_requests==2.32.0.20240712",
     "types-protobuf==5.27.0.20240626",
     "deptry==0.19.1"

requirements.txt

@@ -33,7 +33,7 @@ pyelftools==0.31
 pygments==2.18.0
 python-flirt==0.8.10
 pyyaml==6.0.2
-rich==13.7.1
+rich==13.8.0
 ruamel-yaml==0.18.6
 ruamel-yaml-clib==0.2.8
 setuptools==70.0.0