speedup value import

LDO-CERT · Sep 19, 2024 · 81e446a · 81e446a
1 parent aed7175
commit 81e446a
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 11 deletions.
diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile
@@ -1,4 +1,4 @@
-FROM daskdev/dask:2024.8.2-py3.12
+FROM daskdev/dask:2024.9.0-py3.12
 ENV DEBIAN_FRONTEND noninteractive
 
 ARG local_folder=/uploads
@@ -27,7 +27,7 @@ RUN freshclam
 # Workers should have similar reqs as django
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
+RUN pip install uv==0.4.12 -e git+https://github.com/dadokkio/volatility3.git@bb6ab45363b834576cf392e5f438a9a5f605610f#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh

diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile
@@ -44,7 +44,7 @@ RUN /usr/local/go/bin/go build
 FROM common-base
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
+RUN pip install uv==0.4.12 -e git+https://github.com/dadokkio/volatility3.git@bb6ab45363b834576cf392e5f438a9a5f605610f#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py

diff --git a/orochi/templates/website/index.html b/orochi/templates/website/index.html
@@ -28,7 +28,7 @@ <h1 class="offcanvas-title" id="sidemenuLabel">
         </button>
         {% endif %}
     </span>
-    <input type="text" id="filter_dump" name="filter_dump" style="width: 60%;" class="form-control form-control-sm" placeholder="Filter">
+    <input type="text" id="filter_dump" name="filter_dump" style="width: 60%; float: right;" class="form-control form-control-sm" placeholder="Filter">
     </h6>
     {% include "website/partial_indices.html" %}
     <hr />

diff --git a/orochi/website/admin.py b/orochi/website/admin.py
@@ -16,6 +16,7 @@
 from guardian.shortcuts import assign_perm, get_perms, remove_perm
 from import_export import fields, resources
 from import_export.admin import ExportActionMixin, ImportExportModelAdmin
+from import_export.mixins import BaseImportExportMixin
 from import_export.widgets import ForeignKeyWidget
 
 from orochi.website.defaults import RESULT
@@ -136,6 +137,10 @@ class Meta:
         model = Value
         import_id_fields = ("result", "value")
         exclude = ("id",)
+        skip_diff = True
+        use_bulk = True
+        batch_size = 10000
+        force_init_instance = True
 
 
 @admin.register(Value)
@@ -144,6 +149,7 @@ class ValueAdmin(ImportExportModelAdmin):
     search_fields = ("result__dump__name", "result__plugin__name")
     resource_classes = [ValueResource]
     export_form_class = SelectDumpExportForm
+    skip_import_confirm = True
     formfield_overrides = {
         JSONField: {"widget": JSONEditorWidget},
     }

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,5 +1,5 @@
 # https://github.com/stub42/pytz
-pytz==2024.1
+pytz==2024.2
 # https://github.com/python-pillow/Pillow
 Pillow==10.4.0
 # https://github.com/hynek/argon2_cffi
@@ -71,9 +71,9 @@ django-json-widget==2.0.1
 # Dask & co
 # ------------------------------------------------------------------------------
 # https://github.com/dask/dask
-dask==2024.8.2
+dask==2024.9.0
 # https://github.com/dask/distributed
-distributed==2024.8.2
+distributed==2024.9.0
 # https://msgpack.org/ TO BE ALIGNED WITH SCHEDULER
 msgpack==1.1.0
 # https://github.com/python-lz4/python-lz4
@@ -91,7 +91,7 @@ pandas==2.2.2
 
 # Plotting
 # ------------------------------------------------------------------------------
-plotly==5.24.0
+plotly==5.24.1
 
 # Volatility
 # ------------------------------------------------------------------------------
@@ -100,7 +100,7 @@ plotly==5.24.0
 # https://github.com/Viicos/clamdpy
 clamdpy==0.1.0.post1
 # https://github.com/VirusTotal/vt-py
-vt-py==0.18.3
+vt-py==0.18.4
 # https://github.com/mkorman90/regipy/
 regipy[full]==5.0.0
 # http://www.capstone-engine.org/
@@ -135,7 +135,7 @@ pefile==2024.8.26
 # misp export
 # ------------------------------------------------------------------------------
 # https://github.com/MISP/PyMISP
-pymisp==2.4.197
+pymisp==2.4.198
 
 # ldap
 # ------------------------------------------------------------------------------

diff --git a/requirements/local.txt b/requirements/local.txt
@@ -7,7 +7,7 @@ mypy==1.11.2
 # https://github.com/typeddjango/django-stubs
 django-stubs==5.0.4
 # https://github.com/pytest-dev/pytest
-pytest==8.3.2
+pytest==8.3.3
 # https://github.com/Frozenball/pytest-sugar
 pytest-sugar==1.0.0