diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3968d993f..2dbd64e99 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.14 +current_version = 2.4.16 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P.*)-(?P\d+))? diff --git a/README.md b/README.md index 232abcd46..04514642c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

Incident Response Investigation System
- Current Version v2.4.14 + Current Version v2.4.16
Online Demonstration

@@ -52,7 +52,7 @@ git clone https://github.com/dfir-iris/iris-web.git cd iris-web # Checkout to the last tagged version -git checkout v2.4.14 +git checkout v2.4.16 # Copy the environment file cp .env.model .env diff --git a/deploy/kubernetes/charts/templates/postgres.yaml b/deploy/kubernetes/charts/templates/postgres.yaml index fa3e6066d..a0ea62ae1 100644 --- a/deploy/kubernetes/charts/templates/postgres.yaml +++ b/deploy/kubernetes/charts/templates/postgres.yaml @@ -59,7 +59,7 @@ spec: value: {{ .Values.postgres.POSTGRES_DB | quote }} - name: POSTGRES_USER # Setting Database username - value: {{ .Values.postgres.POSTGRES_ADMIN_USER | quote }} + value: {{ .Values.postgres.POSTGRES_USER | quote }} - name: POSTGRES_PASSWORDD # Setting Database password value: {{ .Values.postgres.POSTGRES_PASSWORD | quote }} @@ -101,4 +101,4 @@ spec: - port: {{ .Values.postgres.service.port }} selector: app: {{ .Values.postgres.app }} ---- \ No newline at end of file +--- diff --git a/docker-compose.yml b/docker-compose.yml index 42917bf03..b9b24a3f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,25 +25,25 @@ services: extends: file: docker-compose.base.yml service: db - image: ${DB_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_db}:${DB_IMAGE_TAG:-v2.4.14} + image: ${DB_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_db}:${DB_IMAGE_TAG:-v2.4.16} app: extends: file: docker-compose.base.yml service: app - image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.14} + image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.16} worker: extends: file: docker-compose.base.yml service: worker - image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.14} + image: ${APP_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_app}:${APP_IMAGE_TAG:-v2.4.16} nginx: extends: file: docker-compose.base.yml service: nginx - image: ${NGINX_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_nginx}:${NGINX_IMAGE_TAG:-v2.4.14} + image: ${NGINX_IMAGE_NAME:-ghcr.io/dfir-iris/iriswebapp_nginx}:${NGINX_IMAGE_TAG:-v2.4.16} volumes: iris-downloads: diff --git a/source/app/alembic/alembic_utils.py b/source/app/alembic/alembic_utils.py index f1f2e9bda..285ae664d 100644 --- a/source/app/alembic/alembic_utils.py +++ b/source/app/alembic/alembic_utils.py @@ -30,4 +30,14 @@ def _has_table(table_name): ) inspector = reflection.Inspector.from_engine(engine) tables = inspector.get_table_names() - return table_name in tables \ No newline at end of file + return table_name in tables + + +def index_exists(table_name, index_name): + config = op.get_context().config + engine = engine_from_config( + config.get_section(config.config_ini_section), prefix="sqlalchemy." + ) + inspector = reflection.Inspector.from_engine(engine) + indexes = inspector.get_indexes(table_name) + return any(index['name'] == index_name for index in indexes) \ No newline at end of file diff --git a/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py b/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py new file mode 100644 index 000000000..6fe3951d4 --- /dev/null +++ b/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py @@ -0,0 +1,66 @@ +"""Add alerts indexes + +Revision ID: d5a720d1b99b +Revises: 11aa5b725b8e +Create Date: 2024-10-28 12:54:22.782313 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy import and_, or_, text +from sqlalchemy.orm import Session + +from app.alembic.alembic_utils import _has_table, index_exists + +# revision identifiers, used by Alembic. +revision = 'd5a720d1b99b' +down_revision = '11aa5b725b8e' +branch_labels = None +depends_on = None + + +def upgrade(): + # Adding indexes to the Alerts table + if _has_table('alerts'): + if not index_exists('alerts', 'idx_alerts_title'): + op.create_index('idx_alerts_title', 'alerts', ['alert_title']) + if not index_exists('alerts', 'idx_alerts_creation_time'): + op.create_index('idx_alerts_creation_time', 'alerts', ['alert_creation_time']) + if not index_exists('alerts', 'idx_alerts_source_event_time'): + op.create_index('idx_alerts_source_event_time', 'alerts', ['alert_source_event_time']) + if not index_exists('alerts', 'idx_alerts_customer_id'): + op.create_index('idx_alerts_customer_id', 'alerts', ['alert_customer_id']) + if not index_exists('alerts', 'alert_source_ref'): + op.create_index('idx_alert_source_ref', 'alerts', ['alert_source_ref']) + + # Adding indexes to the Ioc table + if _has_table('ioc'): + if not index_exists('ioc', 'idx_ioc_value_hash'): + # Create an index on the MD5 hash of ioc_value to handle large values + op.execute(text("CREATE INDEX idx_ioc_value_hash ON ioc (md5(ioc_value::text))")) + if not index_exists('ioc', 'idx_ioc_tags'): + op.create_index('idx_ioc_tags', 'ioc', ['ioc_tags']) + + # Adding indexes to the CaseAssets table + if _has_table('case_assets'): + if not index_exists('case_assets', 'idx_case_assets_name'): + op.create_index('idx_case_assets_name', 'case_assets', ['asset_name']) + if not index_exists('case_assets', 'idx_case_assets_case_id'): + op.create_index('idx_case_assets_case_id', 'case_assets', ['case_id']) + if not index_exists('case_assets', 'idx_case_assets_date_added'): + op.create_index('idx_case_assets_date_added', 'case_assets', ['date_added']) + if not index_exists('case_assets', 'idx_case_assets_date_update'): + op.create_index('idx_case_assets_date_update', 'case_assets', ['date_update']) + + +def downgrade(): + # Drop indexes + op.drop_index('ix_alert_similarity_alert_id', table_name='alert_similarity') + op.drop_index('ix_alert_similarity_similar_alert_id', table_name='alert_similarity') + op.drop_index('ix_alert_similarity_matching_asset_id', table_name='alert_similarity') + op.drop_index('ix_alert_similarity_matching_ioc_id', table_name='alert_similarity') + op.drop_index('ix_alert_similarity_similarity_type', table_name='alert_similarity') + + # Drop AlertSimilarity table + op.drop_table('alert_similarity') + diff --git a/source/app/blueprints/alerts/alerts_routes.py b/source/app/blueprints/alerts/alerts_routes.py index 5b0d540b0..5876f6e1d 100644 --- a/source/app/blueprints/alerts/alerts_routes.py +++ b/source/app/blueprints/alerts/alerts_routes.py @@ -27,7 +27,8 @@ import app from app import db from app.blueprints.case.case_comments import case_comment_update -from app.datamgmt.alerts.alerts_db import get_filtered_alerts, get_alert_by_id, create_case_from_alert +from app.datamgmt.alerts.alerts_db import get_filtered_alerts, get_alert_by_id, create_case_from_alert, \ + register_related_alerts, delete_related_alerts_cache from app.datamgmt.alerts.alerts_db import merge_alert_in_case, unmerge_alert_from_case, cache_similar_alert from app.datamgmt.alerts.alerts_db import get_related_alerts, get_related_alerts_details from app.datamgmt.alerts.alerts_db import get_alert_comments, delete_alert_comment, get_alert_comment @@ -38,7 +39,7 @@ from app.iris_engine.access_control.utils import ac_set_new_case_access from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models.alerts import AlertStatus +from app.models.alerts import AlertStatus, AlertSimilarity, Alert from app.models.authorization import Permissions from app.schema.marshables import AlertSchema, CaseSchema, CommentSchema, CaseAssetsSchema, IocSchema from app.util import ac_api_requires @@ -109,13 +110,12 @@ def alerts_list_route() -> Response: except ValueError: return response_error('Invalid alert ioc') - alert_schema = AlertSchema() - filtered_data = get_filtered_alerts( start_date=request.args.get('creation_start_date'), end_date=request.args.get('creation_end_date'), source_start_date=request.args.get('source_start_date'), source_end_date=request.args.get('source_end_date'), + source_reference=request.args.get('source_reference'), title=request.args.get('alert_title'), description=request.args.get('alert_description'), status=request.args.get('alert_status_id', type=int), @@ -139,15 +139,7 @@ def alerts_list_route() -> Response: if filtered_data is None: return response_error('Filtering error') - alerts = { - 'total': filtered_data.total, - 'alerts': alert_schema.dump(filtered_data.items, many=True), - 'last_page': filtered_data.pages, - 'current_page': filtered_data.page, - 'next_page': filtered_data.next_num if filtered_data.has_next else None, - } - - return response_success(data=alerts) + return response_success(data=filtered_data) @alerts_blueprint.route('/alerts/add', methods=['POST']) @@ -203,6 +195,8 @@ def alerts_add_route() -> Response: cache_similar_alert(new_alert.alert_customer_id, assets=assets_list, iocs=iocs_list, alert_id=new_alert.alert_id, creation_date=new_alert.alert_source_event_time) + + register_related_alerts(new_alert, assets_list=assets, iocs_list=iocs) new_alert = call_modules_hook('on_postload_alert_create', data=new_alert) @@ -360,6 +354,9 @@ def alerts_update_route(alert_id) -> Response: if data.get('alert_owner_id') is None and updated_alert.alert_owner_id is None: updated_alert.alert_owner_id = current_user.id + if data.get('alert_owner_id') == "-1" or data.get('alert_owner_id') == -1: + updated_alert.alert_owner_id = None + # Save the changes db.session.commit() @@ -437,6 +434,9 @@ def alerts_batch_update_route() -> Response: if not user_has_client_access(current_user.id, alert.alert_customer_id): return response_error('User not entitled to update alerts for the client', status=403) + if data.get('alert_owner_id') == "-1" or data.get('alert_owner_id') == -1: + updates['alert_owner_id'] = None + # Deserialize the JSON data into an Alert object alert_schema.load(updates, instance=alert, partial=True) @@ -530,6 +530,9 @@ def alerts_delete_route(alert_id) -> Response: # Delete the case association delete_similar_alert_cache(alert_id=alert_id) + # Delete the similarity entries + delete_related_alerts_cache(alert_id=alert_id) + # Delete the alert from the database db.session.delete(alert) db.session.commit() diff --git a/source/app/blueprints/alerts/templates/alerts.html b/source/app/blueprints/alerts/templates/alerts.html index c2a4d6cbe..a7c28c1ac 100644 --- a/source/app/blueprints/alerts/templates/alerts.html +++ b/source/app/blueprints/alerts/templates/alerts.html @@ -5,6 +5,8 @@ {% block stylesheets %} + + {% endblock stylesheets %} @@ -158,6 +160,10 @@ +
+ + +
diff --git a/source/app/blueprints/case/case_assets_routes.py b/source/app/blueprints/case/case_assets_routes.py index f3761c1ef..e7409e113 100644 --- a/source/app/blueprints/case/case_assets_routes.py +++ b/source/app/blueprints/case/case_assets_routes.py @@ -31,7 +31,7 @@ import app from app import db from app.blueprints.case.case_comments import case_comment_update -from app.datamgmt.case.case_assets_db import add_comment_to_asset +from app.datamgmt.case.case_assets_db import add_comment_to_asset, get_raw_assets from app.datamgmt.case.case_assets_db import create_asset from app.datamgmt.case.case_assets_db import delete_asset from app.datamgmt.case.case_assets_db import delete_asset_comment @@ -94,6 +94,47 @@ def case_assets(caseid, url_redir): return render_template("case_assets.html", case=case, form=form) +@case_assets_blueprint.route('/case/assets/filter', methods=['GET']) +@ac_api_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access) +def case_filter_assets(caseid): + """ + Returns the list of assets from the case. + :return: A JSON object containing the assets of the case, enhanced with assets seen on other cases. + """ + + # Get all assets objects from the case and the customer id + ret = {} + assets = CaseAssetsSchema().dump(get_raw_assets(caseid), many=True) + customer_id = get_case_client_id(caseid) + + ioc_links_req = get_assets_ioc_links(caseid) + + cache_ioc_link = {} + for ioc in ioc_links_req: + + if ioc.asset_id not in cache_ioc_link: + cache_ioc_link[ioc.asset_id] = [ioc._asdict()] + else: + cache_ioc_link[ioc.asset_id].append(ioc._asdict()) + + cases_access = get_user_cases_fast(current_user.id) + + for a in assets: + a['ioc_links'] = cache_ioc_link.get(a['asset_id']) + + if len(assets) < 300: + # Find similar assets from other cases with the same customer + a['link'] = list(get_similar_assets( + a['asset_name'], a['asset_type_id'], caseid, customer_id, cases_access)) + else: + a['link'] = [] + + ret['assets'] = assets + + ret['state'] = get_assets_state(caseid=caseid) + + return response_success("", data=ret) + @case_assets_blueprint.route('/case/assets/list', methods=['GET']) @ac_api_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access) def case_list_assets(caseid): @@ -102,7 +143,6 @@ def case_list_assets(caseid): :return: A JSON object containing the assets of the case, enhanced with assets seen on other cases. """ - # Get all assets objects from the case and the customer id assets = get_assets(caseid) customer_id = get_case_client_id(caseid) diff --git a/source/app/blueprints/case/case_tasks_routes.py b/source/app/blueprints/case/case_tasks_routes.py index 999500ce3..87456fc93 100644 --- a/source/app/blueprints/case/case_tasks_routes.py +++ b/source/app/blueprints/case/case_tasks_routes.py @@ -256,10 +256,10 @@ def case_edit_task(cur_id, caseid): @case_tasks_blueprint.route('/case/tasks/delete/', methods=['POST']) @ac_api_case_requires(CaseAccessLevel.full_access) def case_delete_task(cur_id, caseid): - call_modules_hook('on_preload_task_delete', data=cur_id, caseid=caseid) task = get_task_with_assignees(task_id=cur_id, case_id=caseid) if not task: return response_error("Invalid task ID for this case") + call_modules_hook('on_preload_task_delete', data=task, caseid=caseid) delete_task(task.id) diff --git a/source/app/blueprints/case/templates/modal_add_case_asset.html b/source/app/blueprints/case/templates/modal_add_case_asset.html index 7b1283899..30f7b353b 100644 --- a/source/app/blueprints/case/templates/modal_add_case_asset.html +++ b/source/app/blueprints/case/templates/modal_add_case_asset.html @@ -20,7 +20,7 @@

{{ "Asset #{}".format(asset.asset_id) if asset.asse

{% endif %} diff --git a/source/app/blueprints/case/templates/modal_add_case_event.html b/source/app/blueprints/case/templates/modal_add_case_event.html index f3ecfb92d..2a6d4fa21 100644 --- a/source/app/blueprints/case/templates/modal_add_case_event.html +++ b/source/app/blueprints/case/templates/modal_add_case_event.html @@ -28,7 +28,7 @@

{% if event.event_id %} Event ID #{{ event.event_i {% endif %} diff --git a/source/app/blueprints/case/templates/modal_add_case_ioc.html b/source/app/blueprints/case/templates/modal_add_case_ioc.html index 410f5fd94..873e3a2f9 100644 --- a/source/app/blueprints/case/templates/modal_add_case_ioc.html +++ b/source/app/blueprints/case/templates/modal_add_case_ioc.html @@ -22,7 +22,7 @@

{% if ioc.ioc_id %}Edit IOC #{{ ioc.ioc_id }}{% els {% endif %} diff --git a/source/app/blueprints/case/templates/modal_add_case_rfile.html b/source/app/blueprints/case/templates/modal_add_case_rfile.html index 9a13341d7..c345ced9d 100644 --- a/source/app/blueprints/case/templates/modal_add_case_rfile.html +++ b/source/app/blueprints/case/templates/modal_add_case_rfile.html @@ -20,7 +20,7 @@

{% if rfile.id %}Edit evidence #{{rfile.id}}{% else {% endif %} diff --git a/source/app/blueprints/case/templates/modal_add_case_task.html b/source/app/blueprints/case/templates/modal_add_case_task.html index 78e03aa62..ec9432ffe 100644 --- a/source/app/blueprints/case/templates/modal_add_case_task.html +++ b/source/app/blueprints/case/templates/modal_add_case_task.html @@ -20,7 +20,7 @@

{% if task.id %} Task ID #{{ task.id }}{% else %} {% endif %} diff --git a/source/app/blueprints/case/templates/modal_note_edit.html b/source/app/blueprints/case/templates/modal_note_edit.html index 64183792d..8cf1d04fa 100644 --- a/source/app/blueprints/case/templates/modal_note_edit.html +++ b/source/app/blueprints/case/templates/modal_note_edit.html @@ -15,7 +15,7 @@

Note #{{ note.note_id }}

diff --git a/source/app/blueprints/login/login_routes.py b/source/app/blueprints/login/login_routes.py index 0d184f8ab..1deaef42a 100644 --- a/source/app/blueprints/login/login_routes.py +++ b/source/app/blueprints/login/login_routes.py @@ -253,16 +253,16 @@ def oidc_authorise(): if user and not user.active: return response_error("User not active in IRIS", 403) - return wrap_login_user(user) + return wrap_login_user(user, is_oidc=True) -def wrap_login_user(user): +def wrap_login_user(user, is_oidc=False): session['username'] = user.user if 'SERVER_SETTINGS' not in app.config: app.config['SERVER_SETTINGS'] = get_server_settings_as_dict() - if app.config['SERVER_SETTINGS']['enforce_mfa']: + if app.config['SERVER_SETTINGS']['enforce_mfa'] is True and is_oidc is False: if "mfa_verified" not in session or session["mfa_verified"] is False: return redirect(url_for('mfa_verify')) diff --git a/source/app/business/cases.py b/source/app/business/cases.py index fe41d1386..10700073e 100644 --- a/source/app/business/cases.py +++ b/source/app/business/cases.py @@ -131,7 +131,7 @@ def delete(case_identifier): raise BusinessProcessingError('Cannot delete a primary case to keep consistency') try: - call_modules_hook('on_preload_case_delete', data=case_identifier, caseid=case_identifier) + call_modules_hook('on_preload_case_delete', data=get_case(case_identifier), caseid=case_identifier) if not delete_case(case_identifier): track_activity(f'tried to delete case {case_identifier}, but it doesn\'t exist', caseid=case_identifier, ctx_less=True) diff --git a/source/app/configuration.py b/source/app/configuration.py index 974f55a3c..3585052fd 100644 --- a/source/app/configuration.py +++ b/source/app/configuration.py @@ -263,7 +263,7 @@ class CeleryConfig: # --------- APP --------- class Config: # Handled by bumpversion - IRIS_VERSION = "v2.4.14" # DO NOT EDIT THIS LINE MANUALLY + IRIS_VERSION = "v2.4.16" # DO NOT EDIT THIS LINE MANUALLY if os.environ.get('IRIS_DEMO_VERSION') is not None and os.environ.get('IRIS_DEMO_VERSION') != 'None': IRIS_VERSION = os.environ.get('IRIS_DEMO_VERSION') diff --git a/source/app/datamgmt/alerts/alerts_db.py b/source/app/datamgmt/alerts/alerts_db.py index 8e60716d4..8d3745adb 100644 --- a/source/app/datamgmt/alerts/alerts_db.py +++ b/source/app/datamgmt/alerts/alerts_db.py @@ -23,9 +23,8 @@ from functools import reduce from operator import and_ from sqlalchemy import desc, asc, func, tuple_, or_ -from sqlalchemy.orm import aliased, make_transient -from sqlalchemy.orm import joinedload -from typing import List, Tuple +from sqlalchemy.orm import aliased, make_transient, selectinload +from typing import List, Tuple, Dict import app from app import db @@ -40,8 +39,9 @@ from app.iris_engine.utils.common import parse_bf_date_format from app.models import Cases, EventCategory, Tags, AssetsType, Comments, CaseAssets, alert_assets_association, \ alert_iocs_association, Ioc, IocLink -from app.models.alerts import Alert, AlertStatus, AlertCaseAssociation, SimilarAlertsCache, AlertResolutionStatus -from app.schema.marshables import EventSchema +from app.models.alerts import Alert, AlertStatus, AlertCaseAssociation, SimilarAlertsCache, AlertResolutionStatus, \ + AlertSimilarity +from app.schema.marshables import EventSchema, AlertSchema from app.util import add_obj_history_entry @@ -57,6 +57,7 @@ def get_filtered_alerts( end_date: str = None, source_start_date: str = None, source_end_date: str = None, + source_reference: str = None, title: str = None, description: str = None, status: int = None, @@ -79,30 +80,8 @@ def get_filtered_alerts( """ Get a list of alerts that match the given filter conditions - args: - start_date (datetime): The start date of the alert creation time - end_date (datetime): The end date of the alert creation time - title (str): The title of the alert - description (str): The description of the alert - status (str): The status of the alert - severity (str): The severity of the alert - owner (str): The owner of the alert - source (str): The source of the alert - tags (str): The tags of the alert - case_id (int): The case id of the alert - client (int): The client id of the alert - classification (int): The classification id of the alert - alert_ids (int): The alert ids - assets (list): The assets of the alert - iocs (list): The iocs of the alert - resolution_status (int): The resolution status of the alert - page (int): The page number - per_page (int): The number of alerts per page - sort (str): The sort order - current_user_id (int): The ID of the current user - returns: - list: A list of alerts that match the given filter conditions + dict: A dictionary containing the total count, alerts, and pagination information """ # Build the filter conditions conditions = [] @@ -134,6 +113,9 @@ def get_filtered_alerts( if resolution_status is not None: conditions.append(Alert.alert_resolution_status_id == resolution_status) + if source_reference is not None: + conditions.append(Alert.alert_source_ref.like(f'%{source_reference}%')) + if owner is not None: if owner == -1: conditions.append(Alert.alert_owner_id.is_(None)) @@ -177,25 +159,32 @@ def get_filtered_alerts( order_func = desc if sort == "desc" else asc - try: + alert_schema = AlertSchema() + try: # Query the alerts using the filter conditions filtered_alerts = db.session.query( Alert ).filter( *conditions ).options( - joinedload(Alert.severity), joinedload(Alert.status), joinedload(Alert.customer), joinedload(Alert.cases), - joinedload(Alert.iocs), joinedload(Alert.assets) + selectinload(Alert.severity), selectinload(Alert.status), selectinload(Alert.customer), selectinload(Alert.cases), + selectinload(Alert.iocs), selectinload(Alert.assets) ).order_by( order_func(Alert.alert_source_event_time) ).paginate(page=page, per_page=per_page, error_out=False) + return { + 'total': filtered_alerts.total, + 'alerts': alert_schema.dump(filtered_alerts, many=True), + 'last_page': filtered_alerts.pages, + 'current_page': filtered_alerts.page, + 'next_page': filtered_alerts.next_num if filtered_alerts.has_next else None, + } + except Exception as e: app.app.logger.exception(f"Error getting alerts: {str(e)}") - filtered_alerts = None - - return filtered_alerts + return None def add_alert( @@ -248,7 +237,7 @@ def get_alert_by_id(alert_id: int) -> Alert: """ return ( db.session.query(Alert) - .options(joinedload(Alert.iocs), joinedload(Alert.assets)) + .options(selectinload(Alert.iocs), selectinload(Alert.assets)) .filter(Alert.alert_id == alert_id) .first() ) @@ -785,6 +774,57 @@ def cache_similar_alert(customer_id, assets, iocs, alert_id, creation_date): db.session.commit() +def register_related_alerts(new_alert=None, assets_list=None, iocs_list=None): + """ + Register related alerts + """ + + + # Step 1: Identify similar alerts based on title, assets, and IOCs + similar_alerts = db.session.query(Alert).filter( + Alert.alert_customer_id == new_alert.alert_customer_id, + Alert.alert_id != new_alert.alert_id, + or_( + Alert.alert_title == new_alert.alert_title, + Alert.assets.any(CaseAssets.asset_name.in_([asset.asset_name for asset in new_alert.assets])), + Alert.iocs.any(Ioc.ioc_value.in_([ioc.ioc_value for ioc in new_alert.iocs])) + ) + ).all() + + # Step 2: Create relationships in the AlertSimilarity table + for similar_alert in similar_alerts: + # Matching on title + if new_alert.alert_title == similar_alert.alert_title: + alert_similarity = AlertSimilarity( + alert_id=new_alert.alert_id, + similar_alert_id=similar_alert.alert_id, + similarity_type="title_match" + ) + db.session.add(alert_similarity) + + # Matching on assets + for asset in new_alert.assets: + if asset in similar_alert.assets: + alert_similarity = AlertSimilarity( + alert_id=new_alert.alert_id, + similar_alert_id=similar_alert.alert_id, + similarity_type="asset_match", + matching_asset_id=asset.asset_id + ) + db.session.add(alert_similarity) + + # Matching on IOCs + for ioc in new_alert.iocs: + if ioc in similar_alert.iocs: + alert_similarity = AlertSimilarity( + alert_id=new_alert.alert_id, + similar_alert_id=similar_alert.alert_id, + similarity_type="ioc_match", + matching_ioc_id=ioc.ioc_id + ) + db.session.add(alert_similarity) + + def delete_similar_alert_cache(alert_id): """ Delete the similar alert cache @@ -799,6 +839,24 @@ def delete_similar_alert_cache(alert_id): db.session.commit() +def delete_related_alerts_cache(alert_id): + """ + Delete the related alerts cache + + args: + alert_id (int): The ID of the alert + + returns: + None + """ + AlertSimilarity.query.filter( + or_( + AlertSimilarity.alert_id == alert_id, + AlertSimilarity.similar_alert_id == alert_id + ) + ).delete() + db.session.commit() + def delete_similar_alerts_cache(alert_ids: List[int]): """ Delete the similar alerts cache diff --git a/source/app/datamgmt/case/case_assets_db.py b/source/app/datamgmt/case/case_assets_db.py index a7ae7eec9..f6bc5d0e9 100644 --- a/source/app/datamgmt/case/case_assets_db.py +++ b/source/app/datamgmt/case/case_assets_db.py @@ -83,6 +83,14 @@ def get_assets(caseid): return assets +def get_raw_assets(caseid): + assets = CaseAssets.query.filter( + CaseAssets.case_id == caseid + ).all() + + return assets + + def get_assets_name(caseid): assets_names = CaseAssets.query.with_entities( CaseAssets.asset_name diff --git a/source/app/datamgmt/case/case_db.py b/source/app/datamgmt/case/case_db.py index 3d9998ba7..7050943a4 100644 --- a/source/app/datamgmt/case/case_db.py +++ b/source/app/datamgmt/case/case_db.py @@ -110,7 +110,7 @@ def get_case_report_template(): Languages.name, CaseTemplateReport.description ).filter(and_( - Languages.id == CaseTemplateReport.report_type_id, + Languages.id == CaseTemplateReport.language_id, ReportType.name == "Investigation" )).join( CaseTemplateReport.report_type diff --git a/source/app/datamgmt/case/case_notes_db.py b/source/app/datamgmt/case/case_notes_db.py index a4d16478c..5871c1e25 100644 --- a/source/app/datamgmt/case/case_notes_db.py +++ b/source/app/datamgmt/case/case_notes_db.py @@ -17,7 +17,6 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from flask_login import current_user from sqlalchemy import and_ -from sqlalchemy.orm import joinedload from app import db from app.datamgmt.manage.manage_attribute_db import get_default_custom_attributes diff --git a/source/app/models/alerts.py b/source/app/models/alerts.py index dffae3436..61cccd2e2 100644 --- a/source/app/models/alerts.py +++ b/source/app/models/alerts.py @@ -2,7 +2,7 @@ import uuid from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy import BigInteger, Table, Boolean +from sqlalchemy import BigInteger, Table, Boolean, String from sqlalchemy import Column from sqlalchemy import DateTime from sqlalchemy import ForeignKey @@ -114,3 +114,19 @@ def __init__(self, customer_id, alert_id, asset_name=None, ioc_value=None, asset self.asset_type_id = asset_type_id self.ioc_type_id = ioc_type_id self.created_at = created_at if created_at else datetime.utcnow() + + +class AlertSimilarity(db.Model): + __tablename__ = 'alert_similarity' + + id = Column(BigInteger, primary_key=True) + alert_id = Column(BigInteger, ForeignKey('alerts.alert_id'), nullable=False) + similar_alert_id = Column(BigInteger, ForeignKey('alerts.alert_id'), nullable=False) + similarity_type = Column(String(255), nullable=True) + matching_asset_id = Column(BigInteger, ForeignKey('case_assets.asset_id'), nullable=True) + matching_ioc_id = Column(BigInteger, ForeignKey('ioc.ioc_id'), nullable=True) + + alert = relationship("Alert", foreign_keys=[alert_id]) + similar_alert = relationship("Alert", foreign_keys=[similar_alert_id]) + matching_asset = relationship("CaseAssets") + matching_ioc = relationship("Ioc") diff --git a/source/app/schema/marshables.py b/source/app/schema/marshables.py index 2ec669f95..94eb41190 100644 --- a/source/app/schema/marshables.py +++ b/source/app/schema/marshables.py @@ -666,6 +666,8 @@ class CaseAssetsSchema(ma.SQLAlchemyAutoSchema): ioc_links: List[int] = fields.List(fields.Integer, required=False) asset_enrichment: str = auto_field('asset_enrichment', required=False) asset_type: AssetTypeSchema = ma.Nested(AssetTypeSchema, required=False) + alerts = fields.Nested('AlertSchema', many=True, exclude=['assets']) + analysis_status = fields.Nested('AnalysisStatusSchema', required=False) class Meta: model = CaseAssets diff --git a/source/app/static/assets/css/atlantis.css b/source/app/static/assets/css/atlantis.css index 4d927e071..e93b39c8b 100644 --- a/source/app/static/assets/css/atlantis.css +++ b/source/app/static/assets/css/atlantis.css @@ -16019,4 +16019,32 @@ td.dt-nowrap { white-space: nowrap } .radial-bg-primary { background: radial-gradient(#013479, #011d40) !important; -} \ No newline at end of file +} + +.notification:not(:empty).notification:not([text='0']) { + background: #c40101; + color: white; + padding: 2px 5px 2px 5px; + border-radius: 50% 50%; + margin-bottom: 3px; +} + + .copy-value { + position: relative; + } + + .copy-btn { + display: none; + position: absolute; + top: 50%; + transform: translateY(-50%); + border: none; + background: transparent; + font-size: 1rem; + color: #007bff; + cursor: pointer; + } + + .copy-value:hover .copy-btn { + display: inline; + } \ No newline at end of file diff --git a/source/app/static/assets/css/img/network/downArrow.png b/source/app/static/assets/css/img/network/downArrow.png new file mode 100644 index 000000000..e77d5e6d4 Binary files /dev/null and b/source/app/static/assets/css/img/network/downArrow.png differ diff --git a/source/app/static/assets/css/img/network/leftArrow.png b/source/app/static/assets/css/img/network/leftArrow.png new file mode 100644 index 000000000..3823536e3 Binary files /dev/null and b/source/app/static/assets/css/img/network/leftArrow.png differ diff --git a/source/app/static/assets/css/img/network/minus.png b/source/app/static/assets/css/img/network/minus.png new file mode 100644 index 000000000..30698076b Binary files /dev/null and b/source/app/static/assets/css/img/network/minus.png differ diff --git a/source/app/static/assets/css/img/network/plus.png b/source/app/static/assets/css/img/network/plus.png new file mode 100644 index 000000000..f7ab2a334 Binary files /dev/null and b/source/app/static/assets/css/img/network/plus.png differ diff --git a/source/app/static/assets/css/img/network/rightArrow.png b/source/app/static/assets/css/img/network/rightArrow.png new file mode 100644 index 000000000..c3a209d8b Binary files /dev/null and b/source/app/static/assets/css/img/network/rightArrow.png differ diff --git a/source/app/static/assets/css/img/network/upArrow.png b/source/app/static/assets/css/img/network/upArrow.png new file mode 100644 index 000000000..8aedced7f Binary files /dev/null and b/source/app/static/assets/css/img/network/upArrow.png differ diff --git a/source/app/static/assets/css/img/network/zoomExtends.png b/source/app/static/assets/css/img/network/zoomExtends.png new file mode 100644 index 000000000..74595c635 Binary files /dev/null and b/source/app/static/assets/css/img/network/zoomExtends.png differ diff --git a/source/app/static/assets/js/iris/alerts.js b/source/app/static/assets/js/iris/alerts.js index 95806161d..02c2db2de 100644 --- a/source/app/static/assets/js/iris/alerts.js +++ b/source/app/static/assets/js/iris/alerts.js @@ -519,7 +519,12 @@ const options = { interaction: { hideEdgesOnDrag: false, tooltipDelay: 100, - zoomView: false + zoomView: false, + navigationButtons: true, + keyboard: { + enabled: true, + bindToWindow: true + } }, height: (window.innerHeight - 400) + "px", clickToUse: true, @@ -983,23 +988,41 @@ function renderAlert(alert, expanded=false, modulesOptionsAlertReq,
` : ''} ${alert.alert_source_link ? `
Source Link:
-
${ +
${ alert.alert_source_link && alert.alert_source_link.startsWith('http') - ? `${alert.alert_source_link}` + ? `${alert.alert_source_link} + ` : 'No valid link provided' }
` : ''} ${alert.alert_source_ref ? `
Source Reference:
-
${alert.alert_source_ref}
+
+ ${alert.alert_source_ref} + +
` : ''} ${alert.alert_source_event_time ? `
Source Event Time:
-
${formatTime(alert.alert_source_event_time)} UTC
+
+ ${formatTime(alert.alert_source_event_time)} UTC + +
` : ''} ${alert.alert_creation_time ? `
IRIS Creation Time:
-
${formatTime(alert.alert_creation_time)} UTC
+
+ ${formatTime(alert.alert_creation_time)} UTC + +
` : ''}
@@ -1089,7 +1112,12 @@ function renderAlert(alert, expanded=false, modulesOptionsAlertReq, .map( (ioc) => ` - ${filterXSS(ioc.ioc_value)} + + ${filterXSS(ioc.ioc_value)} + + ${filterXSS(ioc.ioc_description)} ${ioc.ioc_type ? filterXSS(ioc.ioc_type.type_name) : '-'} ${filterXSS(ioc.ioc_tlp) ? ioc.ioc_tlp : '-'} @@ -1141,6 +1169,12 @@ function renderAlert(alert, expanded=false, modulesOptionsAlertReq, .map( (asset) => ` + + ${asset.asset_name ? filterXSS(asset.asset_name) : '-'} + + ${asset.asset_name ? filterXSS(asset.asset_name) : '-'} ${asset.asset_description ? filterXSS(asset.asset_description) : '-'} ${asset.asset_type ? filterXSS(asset.asset_type.asset_name) : '-'} @@ -1412,6 +1446,11 @@ async function updateAlerts(page, per_page, filters = {}, paging=false){ filterString || queryParams.get('filter_id') ? $('#resetFilters').show() : $('#resetFilters').hide(); alertsContainer.show(); + + $('.copy-btn').off().on('click', function() { + let value = $(this).data('value'); + copy_text_clipboard(value); + }); } $('#alertsPerPage').on('change', (e) => { diff --git a/source/app/static/assets/js/iris/case.asset.js b/source/app/static/assets/js/iris/case.asset.js index e5732cb6b..a455eba15 100644 --- a/source/app/static/assets/js/iris/case.asset.js +++ b/source/app/static/assets/js/iris/case.asset.js @@ -120,7 +120,7 @@ function add_assets() { function get_case_assets() { show_loader(); - get_request_api('/case/assets/list') + get_request_api('/case/assets/filter') .done(function (response) { if (response.status == 'success') { if (response.data != null) { @@ -133,7 +133,17 @@ function get_case_assets() { Table.clear(); Table.rows.add(jsdata.assets); Table.columns.adjust().draw(); - load_menu_mod_options('asset', Table, delete_asset); + load_menu_mod_options('asset', Table, delete_asset, [{ + type: 'option', + title: 'Check Alerts', + multi: false, + iconClass: 'fas fa-bell', + action: function(rows) { + let row = rows[0]; + let asset = get_row_value(row); + window.open(`/alerts?alert_assets=${asset}`, '_blank'); + } + }]); $('[data-toggle="popover"]').popover(); set_last_state(jsdata.state); hide_loader(); @@ -377,32 +387,40 @@ $(document).ready(function(){ if (row.link.length > 0) { let has_compro = false; let datacontent = 'data-content="'; - for (let idx in row.link) { - if (row.link[idx]['asset_compromise_status_id'] === 1) { + + row.link.forEach(link => { + const caseInfo = `Observed `; + const caseLink = `case #${link.case_id} `; + const date = link.case_open_date.replace('00:00:00 GMT', ''); + + if (link.asset_compromise_status_id === 1) { has_compro = true; - datacontent += `Observed as compromised
on case #${row.link[idx]['case_id']} (${row.link[idx]['case_open_date'].replace('00:00:00 GMT', '')}) for the same customer.

`; + datacontent += `${caseInfo} as compromised
on ${caseLink} (${date}) for the same customer.

`; } else { - - datacontent += `Observed as not compromised
on case #${row.link[idx]['case_id']} (${row.link[idx]['case_open_date'].replace('00:00:00 GMT', '')}) for the same customer.

`; + datacontent += `${caseInfo} as not compromised
on ${caseLink} (${date}) for the same customer.

`; } - } - if (has_compro) { - compro += ``; + } + + if (row.alerts.length > 0) { + let alerts_content = ""; + + row.alerts.forEach(alert => { + alerts_content += `#${alert.alert_id} - ${alert.alert_title.replace(/'/g, "'").replace(/"/g, """)}
`; + } ); + alerts_content += `More..`; + - compro += datacontent; - compro += '">'; + compro += ``; } let img = $('') .addClass('mr-2') .css({width: '1.5em', height: '1.5em'}) - .attr('src', '/static/assets/img/graph/' + (row['asset_compromise_status_id'] == 1 ? row['asset_icon_compromised'] : row['asset_icon_not_compromised'])) - .attr('title', row['asset_type']); + .attr('src', '/static/assets/img/graph/' + (row['asset_compromise_status_id'] == 1 ? row['asset_type']['asset_icon_compromised'] : row['asset_type']['asset_icon_not_compromised'])) + .attr('title', row['asset_type']['asset_name']); let link = $('') .attr('href', 'javascript:void(0);') @@ -419,7 +437,7 @@ $(document).ready(function(){ } }, { - "data": "asset_type", + "data": "asset_type.asset_name", "render": function (data, type, row, meta) { if (type === 'display') { data = sanitizeHTML(data);} return data; @@ -483,7 +501,7 @@ $(document).ready(function(){ "data": "analysis_status", "render": function(data, type, row, meta) { if (type === 'display') { - data = sanitizeHTML(data); + data = sanitizeHTML(data['name']); if (data == 'To be done') { flag = 'danger'; } else if (data == 'Started') { diff --git a/source/app/static/assets/js/iris/common.js b/source/app/static/assets/js/iris/common.js index cd51aa1c6..befd2ec56 100644 --- a/source/app/static/assets/js/iris/common.js +++ b/source/app/static/assets/js/iris/common.js @@ -626,7 +626,7 @@ function copy_object_link_md(data_type, node_id){ function copy_text_clipboardb(data){ navigator.clipboard.writeText(fromBinary64(data)).then(function() { - notify_success('Copied!'); + notify_success('Copied'); }, function(err) { notify_error('Can\'t copy link. I printed it in console.'); console.error(err); @@ -635,7 +635,7 @@ function copy_text_clipboardb(data){ function copy_text_clipboard(data){ navigator.clipboard.writeText(data).then(function() { - notify_success('Copied!'); + notify_success('Copied'); }, function(err) { notify_error('Can\'t copy link. I printed it in console.'); console.error(err); @@ -1081,7 +1081,7 @@ function goto_case_number() { } -function load_menu_mod_options(data_type, table, deletion_fn) { +function load_menu_mod_options(data_type, table, deletion_fn, additionalOptions = []) { var actionOptions = { classes: [], contextMenu: { @@ -1105,28 +1105,28 @@ function load_menu_mod_options(data_type, table, deletion_fn) { items: [], }; - datatype_map = { + let datatype_map = { 'task': 'tasks', 'ioc': 'ioc', 'evidence': 'evidences', 'note': 'notes', 'asset': 'assets', 'event': 'timeline/events' - } + }; - get_request_api("/dim/hooks/options/"+ data_type +"/list") + get_request_api("/dim/hooks/options/" + data_type + "/list") .done((data) => { - if(notify_auto_api(data, true)) { + if (notify_auto_api(data, true)) { if (data.data != null) { - jsdata = data.data; + let jsdata = data.data; actionOptions.items.push({ type: 'option', title: 'Share', multi: false, iconClass: 'fas fa-share', - action: function(rows){ - row = rows[0]; + action: function(rows) { + let row = rows[0]; copy_object_link(get_row_id(row)); } }); @@ -1136,8 +1136,8 @@ function load_menu_mod_options(data_type, table, deletion_fn) { title: 'Comment', multi: false, iconClass: 'fas fa-comments', - action: function(rows){ - row = rows[0]; + action: function(rows) { + let row = rows[0]; if (data_type in datatype_map) { comment_element(get_row_id(row), datatype_map[data_type]); } @@ -1149,8 +1149,8 @@ function load_menu_mod_options(data_type, table, deletion_fn) { title: 'Markdown Link', multi: false, iconClass: 'fa-brands fa-markdown', - action: function(rows){ - row = rows[0]; + action: function(rows) { + let row = rows[0]; copy_object_link_md(data_type, get_row_id(row)); } }); @@ -1160,19 +1160,22 @@ function load_menu_mod_options(data_type, table, deletion_fn) { title: 'Copy', multi: false, iconClass: 'fa-regular fa-copy', - action: function(rows){ - row = rows[0]; + action: function(rows) { + let row = rows[0]; copy_text_clipboard(get_row_value(row)); } }); + additionalOptions.forEach(option => { + actionOptions.items.push(option); + }); + actionOptions.items.push({ type: 'divider' }); - jdata_menu_options = jsdata; - for (option in jsdata) { - opt = jsdata[option]; + for (let option in jsdata) { + let opt = jsdata[option]; actionOptions.items.push({ type: 'option', @@ -1181,10 +1184,10 @@ function load_menu_mod_options(data_type, table, deletion_fn) { multiTitle: opt.manual_hook_ui_name, iconClass: 'fas fa-rocket', contextMenuClasses: ['text-dark'], - action: function (rows, de, ke) { + action: function(rows, de, ke) { init_module_processing_wrap(rows, data_type, de[0].outerText); }, - }) + }); } if (deletion_fn !== undefined) { @@ -1198,21 +1201,22 @@ function load_menu_mod_options(data_type, table, deletion_fn) { multi: false, iconClass: 'fas fa-trash', contextMenuClasses: ['text-danger'], - action: function(rows){ - row = rows[0]; + action: function(rows) { + let row = rows[0]; deletion_fn(get_row_id(row)); } }); } - tableActions = table.contextualActions(actionOptions); + let tableActions = table.contextualActions(actionOptions); tableActions.update(); } } - }) + }); } + function get_custom_attributes_fields() { values = Object(); has_error = []; @@ -1696,6 +1700,12 @@ function do_deletion_prompt(message, force_prompt=false) { } } +function escapeHtml(text) { + let parser = new DOMParser(); + let escapedDoc = parser.parseFromString(text, 'text/html'); + return escapedDoc.documentElement.textContent; +} + function toBinary64(string) { const codeUnits = new Uint16Array(string.length); for (let i = 0; i < codeUnits.length; i++) { diff --git a/source/dependencies/iris_webhooks_module-1.0.7-py3-none-any.whl b/source/dependencies/iris_webhooks_module-1.0.7-py3-none-any.whl deleted file mode 100644 index 87914c497..000000000 Binary files a/source/dependencies/iris_webhooks_module-1.0.7-py3-none-any.whl and /dev/null differ diff --git a/source/dependencies/iris_webhooks_module-1.0.8-py3-none-any.whl b/source/dependencies/iris_webhooks_module-1.0.8-py3-none-any.whl new file mode 100644 index 000000000..53886a6df Binary files /dev/null and b/source/dependencies/iris_webhooks_module-1.0.8-py3-none-any.whl differ diff --git a/source/requirements.txt b/source/requirements.txt index a29374dfc..8a70664dd 100644 --- a/source/requirements.txt +++ b/source/requirements.txt @@ -48,6 +48,6 @@ dependencies/iris_evtx-1.2.0-py3-none-any.whl dependencies/iris_check_module-1.0.1-py3-none-any.whl dependencies/iris_vt_module-1.2.1-py3-none-any.whl dependencies/iris_misp_module-1.3.0-py3-none-any.whl -dependencies/iris_webhooks_module-1.0.7-py3-none-any.whl +dependencies/iris_webhooks_module-1.0.8-py3-none-any.whl dependencies/iris_intelowl_module-0.1.0-py3-none-any.whl dependencies/iris_seika_module-1.0.0-py3-none-any.whl \ No newline at end of file