From bb6b8e0c97aaba855e93852c506526f703cefd50 Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Mon, 2 Sep 2024 15:00:09 +0200 Subject: [PATCH] #1102 --- compose/local/dask/Dockerfile | 4 +- compose/local/django/Dockerfile | 2 +- config/settings/base.py | 6 ++ orochi/website/admin.py | 71 ++++++++++++++- orochi/website/apps.py | 3 + orochi/website/forms.py | 11 +++ orochi/website/models.py | 149 +------------------------------- orochi/website/signals.py | 143 ++++++++++++++++++++++++++++++ requirements/base.txt | 13 +-- 9 files changed, 242 insertions(+), 160 deletions(-) create mode 100644 orochi/website/signals.py diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile index 57887e89..b3ebbc30 100644 --- a/compose/local/dask/Dockerfile +++ b/compose/local/dask/Dockerfile @@ -1,4 +1,4 @@ -FROM daskdev/dask:2024.8.1-py3.12 +FROM daskdev/dask:2024.8.2-py3.12 ENV DEBIAN_FRONTEND noninteractive ARG local_folder=/uploads @@ -27,7 +27,7 @@ RUN freshclam # Workers should have similar reqs as django WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.3.5 -e git+https://github.com/dadokkio/volatility3.git@517f46e833648d1232b410436e53e07da429d6f5#egg=volatility3 \ +RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@517f46e833648d1232b410436e53e07da429d6f5#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 3f7f919e..d282c275 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -44,7 +44,7 @@ RUN /usr/local/go/bin/go build FROM common-base WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.3.5 -e git+https://github.com/dadokkio/volatility3.git@517f46e833648d1232b410436e53e07da429d6f5#egg=volatility3 \ +RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@517f46e833648d1232b410436e53e07da429d6f5#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py diff --git a/config/settings/base.py b/config/settings/base.py index 1225ebcf..365c8daa 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -7,6 +7,7 @@ import environ import ldap from django_auth_ldap.config import LDAPSearch +from import_export.formats.base_formats import JSON ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent # orochi/ @@ -70,6 +71,7 @@ "django_admin_listfilter_dropdown", "django_admin_multiple_choice_list_filter", "extra_settings", + "import_export", ] LOCAL_APPS = [ @@ -240,6 +242,7 @@ "loggers": { "distributed": {"level": DEBUG_LEVEL, "handlers": ["console"]}, "django_auth_ldap": {"level": DEBUG_LEVEL, "handlers": ["console"]}, + "import_export": {"level": DEBUG_LEVEL, "handlers": ["console"]}, }, } @@ -266,6 +269,9 @@ # AUTOFIELD DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +# IMPORT/EXPORT +IMPORT_EXPORT_FORMATS = [JSON] + # Channels # ------------------------------------------------------------------------------- ASGI_APPLICATION = "config.routing.application" diff --git a/orochi/website/admin.py b/orochi/website/admin.py index b9dad8d9..7d9e5107 100644 --- a/orochi/website/admin.py +++ b/orochi/website/admin.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.contrib import admin from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -9,13 +10,17 @@ ) from django_file_form.model_admin import FileFormAdmin from django_file_form.models import TemporaryUploadedFile -from guardian.admin import GuardedModelAdmin -from guardian.shortcuts import assign_perm, get_objects_for_user, get_perms, remove_perm +from guardian.admin import GuardedModelAdminMixin +from guardian.shortcuts import assign_perm, get_perms, remove_perm +from import_export import fields, resources +from import_export.admin import ExportActionMixin, ImportExportModelAdmin +from import_export.widgets import ForeignKeyWidget from orochi.website.defaults import RESULT from orochi.website.forms import ( PluginCreateAdminForm, PluginEditAdminForm, + ResultDumpExportForm, UserListForm, ) from orochi.website.models import ( @@ -58,10 +63,40 @@ def get_indexes_names(self, obj): return ", ".join([p.name for p in obj.indexes.all()]) +class ResultResource(resources.ModelResource): + dump = fields.Field( + column_name="dump", + attribute="dump", + widget=ForeignKeyWidget(Dump, field="name"), + ) + + plugin = fields.Field( + column_name="plugin", + attribute="plugin", + widget=ForeignKeyWidget(Plugin, field="name"), + ) + + def __init__(self, **kwargs): + super().__init__() + self.dump_ids = kwargs.get("dump_ids") + + def filter_export(self, queryset, **kwargs): + if self.dump_ids: + return queryset.filter(dump__pk__in=self.dump_ids) + return queryset.all() + + class Meta: + model = Result + import_id_fields = ("dump", "plugin") + exclude = ("id",) + + @admin.register(Result) -class ResultAdmin(admin.ModelAdmin): +class ResultAdmin(ImportExportModelAdmin): list_display = ("dump", "plugin", "result") search_fields = ("dump__name", "plugin__name") + resource_classes = [ResultResource] + export_form_class = ResultDumpExportForm list_filter = ( "dump", ResultListFilter, @@ -69,14 +104,42 @@ class ResultAdmin(admin.ModelAdmin): ("plugin", RelatedDropdownFilter), ) + def get_export_resource_kwargs(self, request, **kwargs): + if export_form := kwargs.get("export_form"): + kwargs.update(dump_ids=export_form.cleaned_data["dump"]) + return kwargs + + +class AuthorWidget(ForeignKeyWidget): + def clean(self, value, row=None, *args, **kwargs): + return ( + get_user_model().objects.get_or_create(name=value) + if value and value != "" + else get_user_model().objects.first() + ) + + +class DumpResource(resources.ModelResource): + author = fields.Field( + column_name="author", + attribute="author", + widget=AuthorWidget(settings.AUTH_USER_MODEL, field="name"), + ) + + class Meta: + model = Dump + import_id_fields = ("name",) + exclude = ("id", "plugins", "folder") + @admin.register(Dump) -class DumpAdmin(GuardedModelAdmin): +class DumpAdmin(ImportExportModelAdmin, GuardedModelAdminMixin, ExportActionMixin): actions = ["assign_to_users", "remove_from_users"] list_display = ("name", "author", "index", "status", "get_auth_users") search_fields = ["author__name", "name", "index"] list_filter = ("author", "status", "created_at") exclude = ("suggested_symbols_path", "regipy_plugins", "banner") + resource_classes = [DumpResource] def get_auth_users(self, obj): auth_users = [ diff --git a/orochi/website/apps.py b/orochi/website/apps.py index 455dd90d..fe019070 100644 --- a/orochi/website/apps.py +++ b/orochi/website/apps.py @@ -3,3 +3,6 @@ class WebsiteConfig(AppConfig): name = "orochi.website" + + def ready(self): + import orochi.website.signals diff --git a/orochi/website/forms.py b/orochi/website/forms.py index f32cb6cc..9b56fc7c 100644 --- a/orochi/website/forms.py +++ b/orochi/website/forms.py @@ -10,6 +10,7 @@ MultipleUploadedFileField, UploadedFileField, ) +from import_export.forms import ExportForm from orochi.utils.plugin_install import plugin_install from orochi.website.defaults import ( @@ -20,6 +21,16 @@ from orochi.website.models import Bookmark, Dump, Folder, Plugin, Result, UserPlugin +###################################### +# EXPORT +###################################### +class ResultDumpExportForm(ExportForm): + dump = forms.ModelMultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + queryset=Dump.objects.all(), + ) + + ###################################### # FOLDERS ###################################### diff --git a/orochi/website/models.py b/orochi/website/models.py index ff2bb90a..c88829c2 100644 --- a/orochi/website/models.py +++ b/orochi/website/models.py @@ -1,30 +1,9 @@ -import os -from datetime import datetime - -from asgiref.sync import async_to_sync -from channels.layers import get_channel_layer from colorfield.fields import ColorField from django.conf import settings -from django.contrib.auth import get_user_model from django.contrib.postgres.fields import ArrayField from django.db import models -from django.db.models.signals import post_save, pre_save -from django.dispatch import receiver -from guardian.shortcuts import assign_perm, get_users_with_perms - -from orochi.website.defaults import ( - DEFAULT_YARA_PATH, - RESULT, - RESULT_STATUS_DISABLED, - RESULT_STATUS_NOT_STARTED, - SERVICES, - STATUS, - TOAST_DUMP_COLORS, - TOAST_RESULT_COLORS, - IconEnum, - OSEnum, -) -from orochi.ya.models import Ruleset + +from orochi.website.defaults import RESULT, SERVICES, STATUS, IconEnum, OSEnum class Service(models.Model): @@ -177,127 +156,3 @@ class CustomRule(models.Model): public = models.BooleanField(default=False) path = models.CharField(max_length=255) default = models.BooleanField(default=False) - - -@receiver(post_save, sender=Dump) -def set_permission(sender, instance, created, **kwargs): - """Add object specific permission to the author""" - if created: - assign_perm( - "website.can_see", - instance.author, - instance, - ) - - -@receiver(post_save, sender=get_user_model()) -def get_plugins(sender, instance, created, **kwargs): - if created: - UserPlugin.objects.bulk_create( - [ - UserPlugin(user=instance, plugin=plugin) - for plugin in Plugin.objects.all() - ] - ) - Ruleset.objects.create( - name=f"{instance.username}-Ruleset", - user=instance, - description="Your crafted ruleset", - ) - if os.path.exists(DEFAULT_YARA_PATH): - CustomRule.objects.create( - user=instance, - path=DEFAULT_YARA_PATH, - default=True, - name="DEFAULT", - ) - - -@receiver(post_save, sender=Plugin) -def new_plugin(sender, instance, created, **kwargs): - if created: - # Add new plugin in old dump - for dump in Dump.objects.all(): - if instance.operating_system in [dump.operating_system, "Other"]: - up, created = Result.objects.get_or_create(dump=dump, plugin=instance) - up.result = RESULT_STATUS_NOT_STARTED - up.save() - - # Add new plugin to user - for user in get_user_model().objects.all(): - up, created = UserPlugin.objects.get_or_create(user=user, plugin=instance) - - -@staticmethod -@receiver(pre_save, sender=Dump) -def cache_previous_status(sender, instance, *args, **kwargs): - original_status = None - if instance.id: - original_status = Dump.objects.get(pk=instance.id).status - instance.__original_status = original_status - - -@staticmethod -@receiver(post_save, sender=Dump) -def dump_saved(sender, instance, created, **kwargs): - users = get_users_with_perms(instance, only_with_perms_in=["can_see"]) - if created: - message = f"Dump {instance.name} has been created" - elif instance.__original_status != instance.status: - message = f"Dump {instance.name} has been updated." - else: - return - - message = f"{datetime.now()} || {message}
Status: {instance.get_status_display()}" - - for user in users: - # Send message to room group - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - f"chat_{user.pk}", - { - "type": "chat_message", - "message": message, - }, - ) - - -@staticmethod -@receiver(pre_save, sender=Result) -def cache_previous_result(sender, instance, *args, **kwargs): - original_result = None - if instance.id: - original_result = Result.objects.get(pk=instance.id).result - instance.__original_result = original_result - - -@staticmethod -@receiver(post_save, sender=Result) -def result_saved(sender, instance, created, **kwargs): - dump = instance.dump - users = get_users_with_perms(dump, only_with_perms_in=["can_see"]) - if instance.result in [RESULT_STATUS_DISABLED, RESULT_STATUS_NOT_STARTED]: - return - if created: - message = ( - f"Plugin {instance.plugin.name} on {instance.dump.name} has been created" - ) - elif instance.__original_result != instance.result: - message = ( - f"Plugin {instance.plugin.name} on {instance.dump.name} has been updated" - ) - else: - return - - message = f"{datetime.now()} || {message}
Status: {instance.get_result_display()}" - - for user in users: - # Send message to room group - channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - f"chat_{user.pk}", - { - "type": "chat_message", - "message": message, - }, - ) diff --git a/orochi/website/signals.py b/orochi/website/signals.py new file mode 100644 index 00000000..702e077c --- /dev/null +++ b/orochi/website/signals.py @@ -0,0 +1,143 @@ +import os +from datetime import datetime + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer +from django.contrib.auth import get_user_model +from django.db.models.signals import post_save, pre_save +from django.dispatch import receiver +from guardian.shortcuts import assign_perm, get_users_with_perms + +from orochi.website.defaults import ( + DEFAULT_YARA_PATH, + RESULT_STATUS_DISABLED, + RESULT_STATUS_NOT_STARTED, + TOAST_DUMP_COLORS, + TOAST_RESULT_COLORS, +) +from orochi.website.models import CustomRule, Dump, Plugin, Result, UserPlugin +from orochi.ya.models import Ruleset + + +@receiver(post_save, sender=Dump) +def set_permission(sender, instance, created, **kwargs): + """Add object specific permission to the author""" + if created: + assign_perm( + "website.can_see", + instance.author, + instance, + ) + + +@receiver(post_save, sender=get_user_model()) +def get_plugins(sender, instance, created, **kwargs): + if created: + UserPlugin.objects.bulk_create( + [ + UserPlugin(user=instance, plugin=plugin) + for plugin in Plugin.objects.all() + ] + ) + Ruleset.objects.create( + name=f"{instance.username}-Ruleset", + user=instance, + description="Your crafted ruleset", + ) + if os.path.exists(DEFAULT_YARA_PATH): + CustomRule.objects.create( + user=instance, + path=DEFAULT_YARA_PATH, + default=True, + name="DEFAULT", + ) + + +@receiver(post_save, sender=Plugin) +def new_plugin(sender, instance, created, **kwargs): + if created: + # Add new plugin in old dump + for dump in Dump.objects.all(): + if instance.operating_system in [dump.operating_system, "Other"]: + up, created = Result.objects.get_or_create(dump=dump, plugin=instance) + up.result = RESULT_STATUS_NOT_STARTED + up.save() + + # Add new plugin to user + for user in get_user_model().objects.all(): + up, created = UserPlugin.objects.get_or_create(user=user, plugin=instance) + + +@staticmethod +@receiver(pre_save, sender=Dump) +def cache_previous_status(sender, instance, *args, **kwargs): + original_status = None + if instance.id: + original_status = Dump.objects.get(pk=instance.id).status + instance.__original_status = original_status + + +@staticmethod +@receiver(post_save, sender=Dump) +def dump_saved(sender, instance, created, **kwargs): + users = get_users_with_perms(instance, only_with_perms_in=["can_see"]) + if created: + message = f"Dump {instance.name} has been created" + elif instance.__original_status != instance.status: + message = f"Dump {instance.name} has been updated." + else: + return + + message = f"{datetime.now()} || {message}
Status: {instance.get_status_display()}" + + for user in users: + # Send message to room group + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + f"chat_{user.pk}", + { + "type": "chat_message", + "message": message, + }, + ) + + +@staticmethod +@receiver(pre_save, sender=Result) +def cache_previous_result(sender, instance, *args, **kwargs): + original_result = None + if instance.id: + original_result = Result.objects.get(pk=instance.id).result + instance.__original_result = original_result + + +@staticmethod +@receiver(post_save, sender=Result) +def result_saved(sender, instance, created, **kwargs): + dump = instance.dump + users = get_users_with_perms(dump, only_with_perms_in=["can_see"]) + if instance.result in [RESULT_STATUS_DISABLED, RESULT_STATUS_NOT_STARTED]: + return + if created: + message = ( + f"Plugin {instance.plugin.name} on {instance.dump.name} has been created" + ) + elif instance.__original_result != instance.result: + message = ( + f"Plugin {instance.plugin.name} on {instance.dump.name} has been updated" + ) + else: + return + + message = f"{datetime.now()} || {message}
Status: {instance.get_result_display()}" + + for user in users: + # Send message to room group + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + f"chat_{user.pk}", + { + "type": "chat_message", + "message": message, + }, + ) diff --git a/requirements/base.txt b/requirements/base.txt index a997ab8d..7820496a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -30,14 +30,15 @@ channels_redis==4.2.0 # https://github.com/joke2k/django-environ django-environ==0.11.2 # https://github.com/pennersr/django-allauth -django-allauth[mfa]==64.1.0 +django-allauth[mfa]==64.2.0 # https://github.com/django-crispy-forms/django-crispy-forms django-crispy-forms==2.3 # https://github.com/jazzband/django-redis django-redis==5.4.0 # https://gunicorn.org/ gunicorn==23.0.0 - +# https://github.com/django-import-export +django-import-export==4.1.1 # Django Ninja # ------------------------------------------------------------------------------ @@ -77,9 +78,9 @@ luqum==0.13.0 # Dask & co # ------------------------------------------------------------------------------ # https://github.com/dask/dask -dask==2024.8.1 +dask==2024.8.2 # https://github.com/dask/distributed -distributed==2024.8.1 +distributed==2024.8.2 # https://msgpack.org/ TO BE ALIGNED WITH SCHEDULER msgpack==1.0.8 # https://github.com/python-lz4/python-lz4 @@ -97,7 +98,7 @@ pandas==2.2.2 # Plotting # ------------------------------------------------------------------------------ -plotly==5.23.0 +plotly==5.24.0 # Volatility # ------------------------------------------------------------------------------ @@ -125,7 +126,7 @@ GitPython==3.1.43 # https://github.com/frostming/marko marko==2.1.2 # https://github.com/VirusTotal/yara-x -yara_x==0.6.0 +yara_x==0.7.0 # symbols dwarf # ------------------------------------------------------------------------------