[MRG] Merge pull request #630 from dfir-iris/hotfix_2.4.15-beta

Hotfix 2.4.15

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.4.14
+current_version = 2.4.15
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-(?P<release>.*)-(?P<build>\d+))?

README.md

@@ -6,7 +6,7 @@
 <p align="center">
   Incident Response Investigation System
   <br>
-  <i>Current Version v2.4.14</i>
+  <i>Current Version v2.4.15</i>
   <br>
   <a href="https://v200.beta.dfir-iris.org">Online Demonstration</a>
 </p>
@@ -52,7 +52,7 @@ git clone https://github.com/dfir-iris/iris-web.git
 cd iris-web
 
 # Checkout to the last tagged version 
-git checkout v2.4.14
+git checkout v2.4.15
 
 # Copy the environment file 
 cp .env.model .env

docker-compose.yml

@@ -25,25 +25,25 @@ services:
     extends:
       file: docker-compose.base.yml
       service: db
-    image: ${DB_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_db}:${DB_IMAGE_TAG:-v2.4.14}
+    image: ${DB_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_db}:${DB_IMAGE_TAG:-v2.4.15}
 
   app:
     extends:
       file: docker-compose.base.yml
       service: app
-    image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.14}
+    image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.15}
 
   worker:
     extends:
       file: docker-compose.base.yml
       service: worker
-    image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.14}
+    image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.15}
 
   nginx:
     extends:
       file: docker-compose.base.yml
       service: nginx
-    image: ${NGINX_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_nginx}:${NGINX_IMAGE_TAG:-v2.4.14}
+    image: ${NGINX_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_nginx}:${NGINX_IMAGE_TAG:-v2.4.15}
 
 volumes:
   iris-downloads:

source/app/alembic/alembic_utils.py

@@ -30,4 +30,14 @@ def _has_table(table_name):
     )
     inspector = reflection.Inspector.from_engine(engine)
     tables = inspector.get_table_names()
-    return table_name in tables
+    return table_name in tables
+
+
+def index_exists(table_name, index_name):
+    config = op.get_context().config
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section), prefix="sqlalchemy."
+    )
+    inspector = reflection.Inspector.from_engine(engine)
+    indexes = inspector.get_indexes(table_name)
+    return any(index['name'] == index_name for index in indexes)

source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py

@@ -0,0 +1,66 @@
+"""Add alerts indexes
+
+Revision ID: d5a720d1b99b
+Revises: 11aa5b725b8e
+Create Date: 2024-10-28 12:54:22.782313
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import and_, or_, text
+from sqlalchemy.orm import Session
+
+from app.alembic.alembic_utils import _has_table, index_exists
+
+# revision identifiers, used by Alembic.
+revision = 'd5a720d1b99b'
+down_revision = '11aa5b725b8e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Adding indexes to the Alerts table
+    if _has_table('alerts'):
+        if not index_exists('alerts', 'idx_alerts_title'):
+            op.create_index('idx_alerts_title', 'alerts', ['alert_title'])
+        if not index_exists('alerts', 'idx_alerts_creation_time'):
+            op.create_index('idx_alerts_creation_time', 'alerts', ['alert_creation_time'])
+        if not index_exists('alerts', 'idx_alerts_source_event_time'):
+            op.create_index('idx_alerts_source_event_time', 'alerts', ['alert_source_event_time'])
+        if not index_exists('alerts', 'idx_alerts_customer_id'):
+            op.create_index('idx_alerts_customer_id', 'alerts', ['alert_customer_id'])
+        if not index_exists('alerts', 'alert_source_ref'):
+            op.create_index('idx_alert_source_ref', 'alerts', ['alert_source_ref'])
+
+    # Adding indexes to the Ioc table
+    if _has_table('ioc'):
+        if not index_exists('ioc', 'idx_ioc_value_hash'):
+            # Create an index on the MD5 hash of ioc_value to handle large values
+            op.execute(text("CREATE INDEX idx_ioc_value_hash ON ioc (md5(ioc_value::text))"))
+        if not index_exists('ioc', 'idx_ioc_tags'):
+            op.create_index('idx_ioc_tags', 'ioc', ['ioc_tags'])
+
+    # Adding indexes to the CaseAssets table
+    if _has_table('case_assets'):
+        if not index_exists('case_assets', 'idx_case_assets_name'):
+            op.create_index('idx_case_assets_name', 'case_assets', ['asset_name'])
+        if not index_exists('case_assets', 'idx_case_assets_case_id'):
+            op.create_index('idx_case_assets_case_id', 'case_assets', ['case_id'])
+        if not index_exists('case_assets', 'idx_case_assets_date_added'):
+            op.create_index('idx_case_assets_date_added', 'case_assets', ['date_added'])
+        if not index_exists('case_assets', 'idx_case_assets_date_update'):
+            op.create_index('idx_case_assets_date_update', 'case_assets', ['date_update'])
+
+
+def downgrade():
+    # Drop indexes
+    op.drop_index('ix_alert_similarity_alert_id', table_name='alert_similarity')
+    op.drop_index('ix_alert_similarity_similar_alert_id', table_name='alert_similarity')
+    op.drop_index('ix_alert_similarity_matching_asset_id', table_name='alert_similarity')
+    op.drop_index('ix_alert_similarity_matching_ioc_id', table_name='alert_similarity')
+    op.drop_index('ix_alert_similarity_similarity_type', table_name='alert_similarity')
+
+    # Drop AlertSimilarity table
+    op.drop_table('alert_similarity')
+

source/app/blueprints/alerts/alerts_routes.py

@@ -27,7 +27,8 @@
 import app
 from app import db
 from app.blueprints.case.case_comments import case_comment_update
-from app.datamgmt.alerts.alerts_db import get_filtered_alerts, get_alert_by_id, create_case_from_alert
+from app.datamgmt.alerts.alerts_db import get_filtered_alerts, get_alert_by_id, create_case_from_alert, \
+    register_related_alerts, delete_related_alerts_cache
 from app.datamgmt.alerts.alerts_db import merge_alert_in_case, unmerge_alert_from_case, cache_similar_alert
 from app.datamgmt.alerts.alerts_db import get_related_alerts, get_related_alerts_details
 from app.datamgmt.alerts.alerts_db import get_alert_comments, delete_alert_comment, get_alert_comment
@@ -38,7 +39,7 @@
 from app.iris_engine.access_control.utils import ac_set_new_case_access
 from app.iris_engine.module_handler.module_handler import call_modules_hook
 from app.iris_engine.utils.tracker import track_activity
-from app.models.alerts import AlertStatus
+from app.models.alerts import AlertStatus, AlertSimilarity, Alert
 from app.models.authorization import Permissions
 from app.schema.marshables import AlertSchema, CaseSchema, CommentSchema, CaseAssetsSchema, IocSchema
 from app.util import ac_api_requires
@@ -109,13 +110,12 @@ def alerts_list_route() -> Response:
         except ValueError:
             return response_error('Invalid alert ioc')
 
-    alert_schema = AlertSchema()
-
     filtered_data = get_filtered_alerts(
         start_date=request.args.get('creation_start_date'),
         end_date=request.args.get('creation_end_date'),
         source_start_date=request.args.get('source_start_date'),
         source_end_date=request.args.get('source_end_date'),
+        source_reference=request.args.get('source_reference'),
         title=request.args.get('alert_title'),
         description=request.args.get('alert_description'),
         status=request.args.get('alert_status_id', type=int),
@@ -139,15 +139,7 @@ def alerts_list_route() -> Response:
     if filtered_data is None:
         return response_error('Filtering error')
 
-    alerts = {
-        'total': filtered_data.total,
-        'alerts': alert_schema.dump(filtered_data.items, many=True),
-        'last_page': filtered_data.pages,
-        'current_page': filtered_data.page,
-        'next_page': filtered_data.next_num if filtered_data.has_next else None,
-    }
-
-    return response_success(data=alerts)
+    return response_success(data=filtered_data)
 
 
 @alerts_blueprint.route('/alerts/add', methods=['POST'])
@@ -203,6 +195,8 @@ def alerts_add_route() -> Response:
         cache_similar_alert(new_alert.alert_customer_id, assets=assets_list,
                             iocs=iocs_list, alert_id=new_alert.alert_id,
                             creation_date=new_alert.alert_source_event_time)
+
+        register_related_alerts(new_alert, assets_list=assets, iocs_list=iocs)
 
         new_alert = call_modules_hook('on_postload_alert_create', data=new_alert)
 
@@ -360,6 +354,9 @@ def alerts_update_route(alert_id) -> Response:
         if data.get('alert_owner_id') is None and updated_alert.alert_owner_id is None:
             updated_alert.alert_owner_id = current_user.id
 
+        if data.get('alert_owner_id') == "-1" or data.get('alert_owner_id') == -1:
+            updated_alert.alert_owner_id = None
+
         # Save the changes
         db.session.commit()
 
@@ -437,6 +434,9 @@ def alerts_batch_update_route() -> Response:
             if not user_has_client_access(current_user.id, alert.alert_customer_id):
                 return response_error('User not entitled to update alerts for the client', status=403)
 
+            if data.get('alert_owner_id') == "-1" or data.get('alert_owner_id') == -1:
+                updates['alert_owner_id'] = None
+
             # Deserialize the JSON data into an Alert object
             alert_schema.load(updates, instance=alert, partial=True)
 
@@ -530,6 +530,9 @@ def alerts_delete_route(alert_id) -> Response:
         # Delete the case association
         delete_similar_alert_cache(alert_id=alert_id)
 
+        # Delete the similarity entries
+        delete_related_alerts_cache(alert_id=alert_id)
+
         # Delete the alert from the database
         db.session.delete(alert)
         db.session.commit()

source/app/blueprints/alerts/templates/alerts.html

@@ -5,6 +5,8 @@
 {% block stylesheets %}
     <link rel="stylesheet" href="/static/assets/css/suggestags.css">
     <link rel="stylesheet" href="/static/assets/css/alerts.css">
+    <link rel="stylesheet" href="/static/assets/css/vis.graph.css">
+    <link rel="stylesheet" href="/static/assets/css/vis.min.css">
 
 {% endblock stylesheets %}
 
@@ -158,6 +160,10 @@
                         <label for="alert_ids">Alert(s) ID</label>
                         <input class="form-control" id="alert_ids" name="alert_ids">
                     </div>
+                    <div class="col-md-3 form-group">
+                        <label for="alert_ids">Source Reference</label>
+                        <input class="form-control" id="source_reference" name="source_reference">
+                    </div>
                     <div class="col-md-3 form-group">
                         <label for="case_id">Case ID</label>
                         <input type="number" class="form-control" id="case_id" name="case_id">

source/app/blueprints/case/case_assets_routes.py

@@ -31,7 +31,7 @@
 import app
 from app import db
 from app.blueprints.case.case_comments import case_comment_update
-from app.datamgmt.case.case_assets_db import add_comment_to_asset
+from app.datamgmt.case.case_assets_db import add_comment_to_asset, get_raw_assets
 from app.datamgmt.case.case_assets_db import create_asset
 from app.datamgmt.case.case_assets_db import delete_asset
 from app.datamgmt.case.case_assets_db import delete_asset_comment
@@ -94,6 +94,47 @@ def case_assets(caseid, url_redir):
     return render_template("case_assets.html", case=case, form=form)
 
 
+@case_assets_blueprint.route('/case/assets/filter', methods=['GET'])
+@ac_api_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
+def case_filter_assets(caseid):
+    """
+    Returns the list of assets from the case.
+    :return: A JSON object containing the assets of the case, enhanced with assets seen on other cases.
+    """
+
+    # Get all assets objects from the case and the customer id
+    ret = {}
+    assets = CaseAssetsSchema().dump(get_raw_assets(caseid), many=True)
+    customer_id = get_case_client_id(caseid)
+
+    ioc_links_req = get_assets_ioc_links(caseid)
+
+    cache_ioc_link = {}
+    for ioc in ioc_links_req:
+
+        if ioc.asset_id not in cache_ioc_link:
+            cache_ioc_link[ioc.asset_id] = [ioc._asdict()]
+        else:
+            cache_ioc_link[ioc.asset_id].append(ioc._asdict())
+
+    cases_access = get_user_cases_fast(current_user.id)
+
+    for a in assets:
+        a['ioc_links'] = cache_ioc_link.get(a['asset_id'])
+
+        if len(assets) < 300:
+            # Find similar assets from other cases with the same customer
+            a['link'] = list(get_similar_assets(
+                a['asset_name'], a['asset_type_id'], caseid, customer_id, cases_access))
+        else:
+            a['link'] = []
+
+    ret['assets'] = assets
+
+    ret['state'] = get_assets_state(caseid=caseid)
+
+    return response_success("", data=ret)
+
 @case_assets_blueprint.route('/case/assets/list', methods=['GET'])
 @ac_api_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 def case_list_assets(caseid):
@@ -102,7 +143,6 @@ def case_list_assets(caseid):
     :return: A JSON object containing the assets of the case, enhanced with assets seen on other cases.
     """
 
-    # Get all assets objects from the case and the customer id
     assets = get_assets(caseid)
     customer_id = get_case_client_id(caseid)
 

source/app/blueprints/case/templates/modal_add_case_asset.html

@@ -20,7 +20,7 @@ <h4 class="modal-title mr-4">{{ "Asset #{}".format(asset.asset_id) if asset.asse
                          </div>
                          <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ asset.asset_id }}, 'assets')" title="Comments">
                             <span class="btn-label">
-                                <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                                <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                             </span>
                         </button>
                         {% endif %}

source/app/blueprints/case/templates/modal_add_case_event.html

@@ -28,7 +28,7 @@ <h4 class="modal-title mr-4">{% if event.event_id %} Event ID #{{ event.event_i
                     </div>
                     <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ event.event_id }}, 'timeline/events')" title="Comments">
                         <span class="btn-label">
-                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                         </span>
                     </button>
                 {% endif %}

source/app/blueprints/case/templates/modal_add_case_ioc.html

@@ -22,7 +22,7 @@ <h4 class="modal-title mr-4">{% if ioc.ioc_id %}Edit IOC #{{ ioc.ioc_id }}{% els
                      </div>
                      <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ ioc.ioc_id }}, 'ioc')" title="Comments">
                         <span class="btn-label">
-                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                         </span>
                      </button>
                     {% endif %}

source/app/blueprints/case/templates/modal_add_case_rfile.html

@@ -20,7 +20,7 @@ <h4 class="modal-title mr-4">{% if rfile.id %}Edit evidence #{{rfile.id}}{% else
                      </div>
                     <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ rfile.id }}, 'evidences')" title="Comments">
                         <span class="btn-label">
-                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                         </span>
                     </button>
                     {% endif %}

source/app/blueprints/case/templates/modal_add_case_task.html

@@ -20,7 +20,7 @@ <h4 class="modal-title mr-4">{% if task.id %} Task ID #{{ task.id }}{% else %}
                  </div>
                  <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ task.id }}, 'tasks')" title="Comments">
                         <span class="btn-label">
-                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                            <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                         </span>
                 </button>
                 {% endif %}

source/app/blueprints/case/templates/modal_note_edit.html

@@ -15,7 +15,7 @@ <h4 class="modal-title mr-4">Note #{{ note.note_id }}</h4>
                 </button>
                 <button type="button" class="btn bg-transparent btn-xs" onclick="comment_element({{ note.note_id }}, 'notes')" title="Comments">
                     <span class="btn-label">
-                        <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length }}</span>
+                        <i class="fa-solid fa-comments"></i><span class="notification" id="object_comments_number">{{ comments_map|length if comments_map|length > 0 else '' }}</span>
                     </span>
                 </button>
                  <div class="dropdown">

source/app/configuration.py

@@ -263,7 +263,7 @@ class CeleryConfig:
 # --------- APP ---------
 class Config:
     # Handled by bumpversion
-    IRIS_VERSION = "v2.4.14" # DO NOT EDIT THIS LINE MANUALLY
+    IRIS_VERSION = "v2.4.15" # DO NOT EDIT THIS LINE MANUALLY
 
     if os.environ.get('IRIS_DEMO_VERSION') is not None and os.environ.get('IRIS_DEMO_VERSION') != 'None':
         IRIS_VERSION = os.environ.get('IRIS_DEMO_VERSION')