From 0c6eb2f5c8f30f200ca38d02ac348955f333ed44 Mon Sep 17 00:00:00 2001 From: Paul Hutelmyer Date: Tue, 15 Nov 2022 14:51:43 -0500 Subject: [PATCH 1/5] Adding description components --- app/blueprints/strelka.py | 33 ++++++++++++--------------- app/models.py | 9 ++++++-- ui/src/components/SubmissionTable.js | 7 ++++++ ui/src/pages/Dashboard.js | 34 ++++++++++++++++++---------- 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/app/blueprints/strelka.py b/app/blueprints/strelka.py index 7cefdce..03da9f5 100644 --- a/app/blueprints/strelka.py +++ b/app/blueprints/strelka.py @@ -26,6 +26,7 @@ def submitFile(): if file: try: submitted_at = str(datetime.datetime.utcnow()) + submitted_description = request.form['description'] succeeded, response, file_size = submit_file( file, {"source": "fileshot-webui", "user_name": session.get("user_cn")} @@ -50,6 +51,7 @@ def submitFile(): request.remote_addr, request.headers.get("User-Agent"), user_id, + submitted_description, submitted_at, getRequestTime(response), ) @@ -75,8 +77,7 @@ def submitFile(): def getRequestID(response): return ( response["request"]["id"] - if "request" in response - and "id" in response["request"] + if "request" in response and "id" in response["request"] else "" ) @@ -84,8 +85,7 @@ def getRequestID(response): def getRequestTime(response): return ( str(datetime.datetime.fromtimestamp(response["request"]["time"])) - if "request" in response - and "time" in response["request"] + if "request" in response and "time" in response["request"] else "" ) @@ -102,28 +102,22 @@ def getMimeTypes(response): def getScannersRun(response): return ( - response["file"]["scanners"] - if "file" in response - and "scanners" in response["file"] + response["file"]["scanner_list"] + if "file" in response and "scanner_list" in response["file"] else [] ) def getYaraHits(response): return ( - response["scan"]["yara"]["matches"] - if "scan" in response - and "yara" in response["scan"] - and "matches" in response["scan"]["yara"] + response["scan_yara"]["matches"] + if "scan_yara" in response and "matches" in response["scan_yara"] else [] ) def getHashes(response): - hashes = response["scan"]["hash"].copy() \ - if "scan" in response \ - and "hash" in response["scan"] \ - else {} + hashes = response["scan_hash"] if "scan_hash" in response else {} del hashes["elapsed"] return hashes.items() @@ -133,6 +127,8 @@ def getScanStats(): if not session.get('logged_in'): return "unauthenticated", 401 + current_app.logger.info("fetching scan stats") + all_time = db.session.query(FileSubmission).count() thirty_days = ( db.session.query(FileSubmission) @@ -166,12 +162,12 @@ def getScanStats(): def getTimeDelta(days): return datetime.datetime.utcnow() - datetime.timedelta(days) - @strelka.route("/scans/") def getScan(id): if not session.get("logged_in"): return "unauthenticated", 401 + current_app.logger.info("fetching scan by id: %s", id) submission = db.session.query(FileSubmission).options(joinedload(FileSubmission.user)).filter_by(file_id=id).first() if submission is not None: @@ -191,6 +187,7 @@ def view(): if (just_mine): user_id = session["user_id"] + current_app.logger.info("fetching scans for %s from page %s in batches of %s", user_id, page, per_page) submissions = ( FileSubmission.query.options(joinedload(FileSubmission.user)) .filter(FileSubmission.submitted_by_user_id == user_id) @@ -198,6 +195,7 @@ def view(): .paginate(page, per_page, error_out=False) ) else: + current_app.logger.info("fetching all scans from page %s in batches of %s", page, per_page) submissions = ( FileSubmission.query.options(joinedload(FileSubmission.user)) .order_by(FileSubmission.submitted_at.desc()) @@ -216,8 +214,7 @@ def view(): return paginated_ui, 200 - def submissionsToJson(submission): val = submission.as_dict() val["user"] = submission.user.as_dict() - return val \ No newline at end of file + return val diff --git a/app/models.py b/app/models.py index c8752a3..27f5f59 100644 --- a/app/models.py +++ b/app/models.py @@ -6,26 +6,29 @@ class FileSubmission(db.Model): __tablename__ = "file_submission" + # Database Metadata id = db.Column(db.Integer, primary_key=True) + # File Metadata file_id = db.Column(db.String(), unique=True) file_name = db.Column(db.String()) file_size = db.Column(db.Integer()) + # Strelka Metadata strelka_response = db.Column(db.JSON()) mime_types = db.Column(db.ARRAY(db.String(), dimensions=1)) yara_hits = db.Column(db.ARRAY(db.String(), dimensions=1)) scanners_run = db.Column(db.ARRAY(db.String(), dimensions=1)) hashes = db.Column(db.ARRAY(db.String(), dimensions=2)) + # Submission Metadata submitted_from_ip = db.Column(db.String()) submitted_from_client = db.Column(db.String()) - + submitted_description = db.Column(db.String()) submitted_by_user_id = db.Column( db.ForeignKey("user.id"), nullable=False, index=True ) user = relationship("User", back_populates="submissions") - submitted_at = db.Column(db.DateTime(), default=db.func.now(), index=True) processed_at = db.Column(db.DateTime()) @@ -42,6 +45,7 @@ def __init__( submitted_from_ip, submitted_from_client, submitted_by_user_id, + submitted_description, submitted_at, processed_at, ): @@ -56,6 +60,7 @@ def __init__( self.submitted_from_ip = submitted_from_ip self.submitted_from_client = submitted_from_client self.submitted_by_user_id = submitted_by_user_id + self.submitted_description = submitted_description self.submitted_at = submitted_at self.processed_at = processed_at diff --git a/ui/src/components/SubmissionTable.js b/ui/src/components/SubmissionTable.js index 464301c..60f8e4e 100644 --- a/ui/src/components/SubmissionTable.js +++ b/ui/src/components/SubmissionTable.js @@ -126,6 +126,13 @@ const SubmissionTable = ({ filesUploaded, page_size }) => { {full.file_name} ), }, + { + title: "Description", + dataIndex: "submitted_description", + key: "submitted_description", + width: 200, + render: (_, full) =>

{full.submitted_description}

, + }, { title: "Submitted by", dataIndex: "user.user_cn", diff --git a/ui/src/pages/Dashboard.js b/ui/src/pages/Dashboard.js index ad88094..7a8b467 100644 --- a/ui/src/pages/Dashboard.js +++ b/ui/src/pages/Dashboard.js @@ -1,6 +1,7 @@ import React, { useEffect, useState } from "react"; -import { Row, Col, Card, Statistic, Typography, message } from "antd"; +import { Row, Col, Card, Input, Statistic, Typography, message } from "antd"; +import { MessageOutlined } from '@ant-design/icons'; import PageWrapper from "../components/PageWrapper"; import SubmissionTable from "../components/SubmissionTable"; import Dropzone from "../components/Dropzone"; @@ -10,6 +11,7 @@ import { fetchWithTimeout } from "../util"; const { Title, Text } = Typography; const DashboardPage = (props) => { + const [fileDescription, setFileDescription] = useState("No Description Provided"); const [filesUploaded, setFilesUploaded] = useState(0); const [loadingStats, setLoadingStats] = useState(true); const [stats, setStats] = useState({ @@ -19,6 +21,10 @@ const DashboardPage = (props) => { twentyfour_hours: 0, }); + const setDescription = event => { + setFileDescription(event.target.value); + }; + useEffect(() => { let mounted = true; setLoadingStats(true); @@ -50,6 +56,7 @@ const DashboardPage = (props) => { const uploadProps = { name: "file", + data: { description: fileDescription }, withCredentials: true, action: `${APP_CONFIG.BACKEND_URL}/strelka/upload`, onChange(info) { @@ -111,17 +118,20 @@ const DashboardPage = (props) => {
- - - Quick Upload - - Drop a file here to start a Strelka Scan. - - -
- - - + + + Upload File + + Drop a file below and add a description to start a Strelka Scan. + + +
+ }/> +
+
+ + + From 7bbaec886ff71196d7c3747e2b7edc1629f3ff47 Mon Sep 17 00:00:00 2001 From: Paul Hutelmyer Date: Tue, 15 Nov 2022 14:51:51 -0500 Subject: [PATCH 2/5] Removing migrations --- migrations/README | 1 - migrations/alembic.ini | 50 ---------------------- migrations/env.py | 90 --------------------------------------- migrations/script.py.mako | 24 ----------- 4 files changed, 165 deletions(-) delete mode 100644 migrations/README delete mode 100644 migrations/alembic.ini delete mode 100644 migrations/env.py delete mode 100644 migrations/script.py.mako diff --git a/migrations/README b/migrations/README deleted file mode 100644 index 98e4f9c..0000000 --- a/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini deleted file mode 100644 index ec9d45c..0000000 --- a/migrations/alembic.ini +++ /dev/null @@ -1,50 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic,flask_migrate - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[logger_flask_migrate] -level = INFO -handlers = -qualname = flask_migrate - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py deleted file mode 100644 index 42438a5..0000000 --- a/migrations/env.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import with_statement - -import logging -from logging.config import fileConfig - -from flask import current_app - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option( - 'sqlalchemy.url', - str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) -target_metadata = current_app.extensions['migrate'].db.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=target_metadata, literal_binds=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - - connectable = current_app.extensions['migrate'].db.engine - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako deleted file mode 100644 index 2c01563..0000000 --- a/migrations/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} From cfe93bea673eab99ecca068207d0ad024b2c28fb Mon Sep 17 00:00:00 2001 From: Paul Hutelmyer Date: Tue, 15 Nov 2022 14:52:01 -0500 Subject: [PATCH 3/5] Updating docker compose default environment variable --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4cbf70b..e067d31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: networks: - strelkanet environment: - - DATABASE_HOST=postgresdb + - DATABASE_HOST=strelka-ui-postgresdb-1 - DATABASE_NAME=strelka_ui - DATABASE_USERNAME=postgres - DATABASE_PASSWORD=postgres From f344e73cc59ec70e753d91e98063cc6e30620689 Mon Sep 17 00:00:00 2001 From: Paul Hutelmyer Date: Tue, 15 Nov 2022 14:53:23 -0500 Subject: [PATCH 4/5] Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e209dd..b93adab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog Changes to the project will be tracked in this file via the date of change. +## 2022-11-15 +### Added +- Ability to add description to uploaded files + +### Changed +- Changing default docker-compose postgresdb name +- Removing unnecessary directory + ## 2022-11-04 ### Changed - Bug fix for issues when loading scanners with uppercase names From a095fec5819a29ade3883f5684a62bd092ddf0ac Mon Sep 17 00:00:00 2001 From: Paul Hutelmyer Date: Tue, 15 Nov 2022 14:56:06 -0500 Subject: [PATCH 5/5] Update requirements.txt --- app/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/requirements.txt b/app/requirements.txt index b076e11..61e0e81 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -32,7 +32,7 @@ Paste==3.5.0 pathspec==0.9.0 platformdirs==2.4.1 protobuf==3.18.3 -psycopg2==2.8.6 +psycopg2-binary==2.9.4 pyasn1==0.4.8 pycparser==2.20 PyJWT==2.4.0