DataModel Plugin

api_app/analyzers_manager/classes.py

@@ -35,6 +35,28 @@ class BaseAnalyzerMixin(Plugin, metaclass=ABCMeta):
     ObservableTypes = ObservableTypes
     TypeChoices = TypeChoices
 
+    def _do_create_data_model(self) -> bool:
+        return True
+
+    def _create_data_model_mtm(self):
+        return {}
+
+    def _update_data_model(self, data_model) -> None:
+        mtm = self._create_data_model_mtm()
+        for field_name, value in mtm.items():
+            field = getattr(data_model, field_name)
+            field.set(value)
+
+    def create_data_model(self):
+        self.report: AnalyzerReport
+        if self._do_create_data_model():
+            data_model = self.report.create_data_model()
+            if data_model:
+                self._update_data_model(data_model)
+                data_model.save()
+            return data_model
+        return None
+
     @classmethod
     @property
     def config_exception(cls):
@@ -108,7 +130,11 @@ def after_run_success(self, content):
         Args:
             content (any): The content to process after a successful run.
         """
-        super().after_run_success(self._validate_result(content, max_recursion=15))
+        result = super().after_run_success(
+            self._validate_result(content, max_recursion=15)
+        )
+        self.create_data_model()
+        return result
 
 
 class ObservableAnalyzer(BaseAnalyzerMixin, metaclass=ABCMeta):
@@ -326,7 +352,7 @@ def __polling(self, req_key: str, chance: int, re_poll_try: int = 0):
             return self.__polling(req_key, chance, re_poll_try=re_poll_try + 1)
         else:
             status = json_data.get("status", None)
-            if status and status == self._job.Status.RUNNING.value:
+            if status and status == self._job.STATUSES.RUNNING.value:
                 logger.info(
                     f"Poll number #{chance + 1}, "
                     f"status: 'running' <-- {self.__repr__()}"

api_app/analyzers_manager/file_analyzers/elf_info.py

@@ -46,7 +46,7 @@ def run(self):
             )
             logger.warning(warning_message)
             self.report.errors.append(warning_message)
-            self.report.status = self.report.Status.FAILED
+            self.report.status = self.report.STATUSES.FAILED
             self.report.save()
 
         return results

api_app/analyzers_manager/file_analyzers/pe_info.py

@@ -165,7 +165,7 @@ def run(self):
             )
             logger.warning(warning_message)
             self.report.errors.append(warning_message)
-            self.report.status = self.report.Status.FAILED
+            self.report.status = self.report.STATUSES.FAILED
             self.report.save()
 
         return results

api_app/analyzers_manager/file_analyzers/yara_scan.py

@@ -438,3 +438,32 @@ def update(cls):
         logger.info("Finished updating yara rules")
         set_permissions(settings.YARA_RULES_PATH)
         return True
+
+    def _create_data_model_mtm(self):
+        from api_app.data_model_manager.models import Signature
+
+        signatures = []
+        for signature in self.report.report:
+            url = signature.pop("rule_url", None)
+            sign = Signature.objects.create(
+                provider=Signature.PROVIDERS.YARA.value,
+                signature=signature,
+                url=url,
+                score=1,
+            )
+            signatures.append(sign)
+
+        return {"signatures": signatures}
+
+    def _update_data_model(self, data_model):
+        from api_app.data_model_manager.models import FileDataModel
+
+        super()._update_data_model(data_model)
+        if data_model:
+            data_model: FileDataModel
+            signatures = data_model.signatures.count()
+            if signatures > 20:
+                data_model.evaluation = data_model.EVALUATIONS.MALICIOUS.value
+            elif signatures > 10:
+                data_model.evaluation = data_model.EVALUATIONS.SUSPICIOUS.value
+            data_model.save()

api_app/analyzers_manager/migrations/0123_analyzerconfig_mapping_data_model.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.15 on 2024-10-14 07:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzers_manager", "0122_alter_soft_time_limit"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="analyzerconfig",
+            name="mapping_data_model",
+            field=models.JSONField(
+                default=dict, help_text="Mapping data_model_key: analyzer_report_key. "
+            ),
+        ),
+    ]

api_app/analyzers_manager/migrations/0124_data_mapping.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.15 on 2024-10-14 07:24
+
+from django.db import migrations
+
+
+def migrate_urlhaus(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="URLhaus").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "urlhaus_reference": "external_references",
+        "$Malicious": "evaluation",
+        "urls.url": "related_threats",
+    }
+    ac.save()
+
+
+def migrate_maxmind(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="MaxMindGeoIP").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "country_code": "country.iso_code",
+        "registered_country_code": "registered_country_code.iso_code",
+        "asn": "autonomous_system_number",
+        "isp": "autonomous_system_organization",
+    }
+    ac.save()
+
+
+def migrate_abuse_ipdb(apps, schema_editor):
+    AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
+    ac = AnalyzerConfig.objects.filter(name="AbuseIPDB").first()
+    if not ac:
+        return
+    ac.mapping_data_model = {
+        "country_code": "data.countryCode",
+        "external_references": "permalink",
+        "resolutions": "data.hostnames",
+        "isp": "data.isp",
+        "tags": "categories_found",
+    }
+    ac.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzers_manager", "0123_analyzerconfig_mapping_data_model"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_maxmind, migrations.RunPython.noop),
+        migrations.RunPython(migrate_abuse_ipdb, migrations.RunPython.noop),
+        migrations.RunPython(migrate_urlhaus, migrations.RunPython.noop),
+    ]

api_app/analyzers_manager/models.py

@@ -1,12 +1,14 @@
 # This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
 # See the file 'LICENSE' for copying permission.
-
+import json
 from logging import getLogger
-from typing import Optional
+from typing import Dict, Optional, Type
 
 from django.contrib.contenttypes.fields import GenericRelation
+from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.db.models import ForeignKey
 
 from api_app.analyzers_manager.constants import (
     HashChoices,
@@ -16,6 +18,12 @@
 from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException
 from api_app.analyzers_manager.queryset import AnalyzerReportQuerySet
 from api_app.choices import TLP, PythonModuleBasePaths
+from api_app.data_model_manager.models import (
+    BaseDataModel,
+    DomainDataModel,
+    FileDataModel,
+    IPDataModel,
+)
 from api_app.fields import ChoiceArrayField
 from api_app.models import AbstractReport, PythonConfig, PythonModule
 
@@ -32,6 +40,83 @@ class Meta:
         unique_together = [("config", "job")]
         indexes = AbstractReport.Meta.indexes
 
+    @property
+    def data_model_class(self) -> Type[BaseDataModel]:
+        if self.job.is_sample:
+            return FileDataModel
+        if self.job.observable_classification == ObservableTypes.IP.value:
+            return IPDataModel
+        if self.job.observable_classification == ObservableTypes.DOMAIN.value:
+            return DomainDataModel
+        raise NotImplementedError(
+            f"Unable to find data model for {self.job.observable_classification}"
+        )
+
+    def _validation_before_data_model(self) -> bool:
+        if not self.status == self.STATUSES.SUCCESS.value:
+            logger.info(
+                f"Skipping data model of {self.config.name} for job {self.config.pk} because status is "
+                f"{self.status}"
+            )
+            return False
+        data_model_keys = self.data_model_class.get_fields().keys()
+        for data_model_key in self.config.mapping_data_model.values():
+            if data_model_key not in data_model_keys:
+                self.errors.append(
+                    f"Field {data_model_key} not present in {self.data_model_class.__name__}"
+                )
+        return True
+
+    def _create_data_model_dictionary(self) -> Dict:
+        result = {}
+        data_model_fields = self.data_model_class.get_fields()
+        logger.info(f"Mapping is {json.dumps(self.config.mapping_data_model)}")
+        for report_key, data_model_key in self.config.mapping_data_model.items():
+            # this is a constant
+            if report_key.startswith("$"):
+                value = report_key
+            # this is a field of the report
+            else:
+                try:
+                    value = self.get_value(self.report, report_key.split("."))
+                    logger.info(f"Retrieved {value} from key {report_key}")
+                except Exception:
+                    # validation
+                    self.errors.append(f"Field {report_key} not present in report")
+                    continue
+                    # create the related object if necessary
+                if isinstance(data_model_fields[data_model_key], ForeignKey):
+                    # to create an object we need at least
+                    if not isinstance(value, dict):
+                        self.errors.append(
+                            f"Field {report_key} has type {type(report_key)} while a dictionary is expected"
+                        )
+                        continue
+                    value, _ = data_model_fields[
+                        data_model_key
+                    ].related_model.objects.get_or_create(**value)
+                elif isinstance(data_model_fields[data_model_key], ArrayField):
+                    if data_model_key not in result:
+                        result[data_model_key] = []
+                    if isinstance(value, list):
+                        result[data_model_key].extend(value)
+                    elif isinstance(value, dict):
+                        result[data_model_key].extend(list(value.keys()))
+                    else:
+                        result[data_model_key].append(value)
+            result[data_model_key] = value
+        return result
+
+    def create_data_model(self) -> Optional[BaseDataModel]:
+        if not self._validation_before_data_model():
+            return None
+        dictionary = self._create_data_model_dictionary()
+        data_model = self.data_model_class.objects.create(
+            **dictionary, analyzer_report=self
+        )
+
+        return data_model
+
 
 class MimeTypes(models.TextChoices):
     # IMPORTANT! in case you update this Enum remember to update also the frontend
@@ -188,6 +273,10 @@ class AnalyzerConfig(PythonConfig):
     orgs_configuration = GenericRelation(
         "api_app.OrganizationPluginConfiguration", related_name="%(class)s"
     )
+    mapping_data_model = models.JSONField(
+        default=dict,
+        help_text="Mapping analyzer_report_key: data_model_key. Keys preceded by the symbol $ will be considered as constants.",
+    )
 
     @classmethod
     @property

api_app/analyzers_manager/observable_analyzers/abuseipdb.py

@@ -4,6 +4,7 @@
 import requests
 
 from api_app.analyzers_manager.classes import ObservableAnalyzer
+from api_app.analyzers_manager.models import AnalyzerReport
 from tests.mock_utils import MockUpResponse, if_mock_connections, patch
 
 
@@ -93,3 +94,15 @@ def _monkeypatch(cls):
             )
         ]
         return super()._monkeypatch(patches=patches)
+
+    def _update_data_model(self, data_model) -> None:
+        super()._update_data_model(data_model)
+        if self.report.report.get("totalReports", 0):
+            self.report: AnalyzerReport
+            if self.report.report["isWhitelisted"]:
+                evaluation = (
+                    self.report.data_model_class.EVALUATIONS.FALSE_POSITIVE.value
+                )
+            else:
+                evaluation = self.report.data_model_class.EVALUATIONS.MALICIOUS.value
+            data_model.evaluation = evaluation

api_app/analyzers_manager/observable_analyzers/maxmind.py

@@ -228,3 +228,29 @@ def _monkeypatch(cls):
         # completely skip because does not work without connection.
         patches = [if_mock_connections(patch.object(cls, "run", return_value={}))]
         return super()._monkeypatch(patches=patches)
+
+    def _update_data_model(self, data_model) -> None:
+        from api_app.analyzers_manager.models import AnalyzerReport
+
+        super()._update_data_model(data_model)
+        org = self.report.report.get("autonomous_system_organization", None)
+        if org:
+            org = org.lower()
+            self.report: AnalyzerReport
+            if org in ["fastly", "cloudflare", "akamai"]:
+                data_model.evaluation = (
+                    self.report.data_model_class.EVALUATIONS.CLEAN.value
+                )
+            elif org in [
+                "zscaler",
+                "palo alto networks",
+                "microdata service srl",
+                "forcepoint",
+            ]:
+                data_model.evaluation = (
+                    self.report.data_model_class.EVALUATIONS.FALSE_POSITIVE.value
+                )
+            elif org in ["stark industries"]:
+                data_model.evaluation = (
+                    self.report.data_model_class.EVALUATIONS.SUSPICIOUS.value
+                )

api_app/analyzers_manager/observable_analyzers/urlhaus.py

@@ -39,6 +39,12 @@ def run(self):
 
         return response.json()
 
+    def _do_create_data_model(self) -> bool:
+        return (
+            super()._do_create_data_model()
+            and self.report.report.get("query_status", "no_results") != "no_results"
+        )
+
     @classmethod
     def _monkeypatch(cls):
         patches = [