diff --git a/.gitignore b/.gitignore index ba28462..de8c1af 100755 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,7 @@ vendor/* # Agent agent/module.py + +# Editors +.vscode +.mypy_cache diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..3833301 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,45 @@ + +FROM debian:stretch + +WORKDIR /opt + +RUN apt-get update && \ + apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + git \ + python-dev \ + python-pip \ + screen \ + p7zip-full \ + libjpeg-dev \ + zlib1g-dev \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common + +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv 9DA31620334BD75D9DCB49F368818C72E52529D4 +RUN echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | tee /etc/apt/sources.list.d/mongodb-org-4.0.list + +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - && \ + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" + +RUN apt-get update && apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io + +RUN apt-get update && apt-get install -y \ + mongodb-org + + +RUN pip install virtualenv && \ + git clone https://github.com/certsocietegenerale/fame + +ENTRYPOINT ["/opt/fame/docker/launch.sh"] +EXPOSE 4200 diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..a05a531 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,23 @@ +# Docker support + +This is probably the quickest way to spawn a Fame dev instance. + +## Install docker + +Follow the [official instructions](https://www.docker.com/community-edition). + +## Clone the repo + + $ git clone https://github.com/certsocietegenerale/fame/ + $ cd fame/docker + +## Build docker image + + $ docker build -t famedev:latest . + +## Run docker image + +To run container based on famedev image with docker inception (bind docker socket and bind fame temp dir). +So chose a temp directory on your system and use the following command. + + $ docker run -it -v /var/run/docker.sock:/var/run/docker.sock -v :/opt/fame/temp --name famedev -p 4200:4200 famedev:latest diff --git a/docker/launch.sh b/docker/launch.sh new file mode 100755 index 0000000..d93b5f0 --- /dev/null +++ b/docker/launch.sh @@ -0,0 +1,8 @@ +#!/bin/bash +cd /opt/fame +service docker start +mongod -f /etc/mongod.conf & +utils/run.sh utils/install.py +screen -dmS "web" bash -c "utils/run.sh webserver.py" +screen -dmS "worker" bash -c "utils/run.sh worker.py" +/bin/bash diff --git a/docs/modules.rst b/docs/modules.rst index 8c3eb53..dc6f21d 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -31,51 +31,6 @@ The best practice is to do the following: * Create a python file inside your directory for your module. * Inside your python file, create a class that inherits from :class:`fame.core.module.PreloadingModule`, :class:`fame.core.module.ProcessingModule`, :class:`fame.core.module.ReportingModule`, :class:`fame.core.module.ThreatIntelligenceModule` or :class:`fame.core.module.AntivirusModule`. -Writing a Preloading module -=========================== - -Preloading modules are used to download a sample file automatically to make it available to FAME. - -To define a new Preloading Module just create a Python class that inherits from :class:`fame.core.module.PreloadingModule` and implements :func:`fame.core.module.PreloadingModule.preload`. - -These methods should return a boolean indicating if the sample was downloaded successfully. If the return value is ``True``, one thing will happen: - -* A tag with the module's name will automatically be added to the analysis. - -For example, the module `virustotal_download` takes a hash and download the sample from VirusTotal. If it could not download the sample successfully, it should return ``False`` so that the next preloading module can be scheduled. - -Here is the minimal code required for a preloading module:: - - from fame.core.module import PreloadingModule - - - class Dummy(PreloadingModule): - # You have to give your module a name, and this name should be unique. - name = "dummy" - - # (optional) Describe what your module will do. This will be displayed to users. - description = "Does nothing." - - # This method will be called, with the hash of the sample in target - def preload(self, target): - return True - -Scope ------ - -It may happen that an analyst only has a hash available for analysis. In this case, FAME can download the sample from configured sample sources and trigger an analysis of the sample by its own. - -Adding preloading results -------------------------- - -Once the module successfully preloaded the sample for FAME, it must add the result to the analysis. As soon as such result is added, FAME schedules the regular analysis based on what was added to the analysis. - -You can declare a preloading result by calling :func:`fame.core.module.PreloadingModule.add_preloaded_file`. The function expects a filename and a file-like object with the available data. - -Testing Preloading Modules --------------------------- - -See :ref:`testing_processing_modules`. Writing a Processing module =========================== @@ -362,6 +317,45 @@ When it comes to testing your processing modules during development, you have tw * The simpler option is to use the :ref:`single_module` utility. This way, you don't need a webserver, a worker or even a MongoDB instance. * An ``IsolatedProcessingModule`` can also be tested with the :ref:`single_module` utility. By default, it will execute inside a Virtual Machine (as it should). If you want to test your module without this overhead (if you are already inside the VM for example), you can use the ``-l, --local`` switch. +Writing a Preloading module +=========================== + +Preloading modules are used to download a sample file automatically to make it available to FAME. + +To define a new Preloading Module just create a Python class that inherits from :class:`fame.core.module.PreloadingModule` and implements :func:`fame.core.module.PreloadingModule.preload`. + +If the module was able to successfuly find a sample associated with submitted hash, it should call the `add_preloaded_file` method. If this method is not called, the next preloading module will be scheduled. + +For example, the module `virustotal_download` takes a hash and download the sample from VirusTotal. + +Here is the minimal code required for a preloading module:: + + from fame.core.module import PreloadingModule + + + class Dummy(PreloadingModule): + # You have to give your module a name, and this name should be unique. + name = "dummy" + + # (optional) Describe what your module will do. This will be displayed to users. + description = "Does nothing." + + # This method will be called, with the hash of the sample in target + def preload(self, target): + return False + +Scope +----- + +It may happen that an analyst only has a hash available for analysis. In this case, FAME can download the sample from configured sample sources and trigger an analysis of the sample by its own. + +Adding the preloading result +---------------------------- + +Once the module successfully preloaded the sample for FAME, it must add the file to the analysis. Based on what type the file is, FAME then schedules suitable processing modules (if magic mode is enabled). + +You can add a preloaded file by calling :func:`fame.core.module.PreloadingModule.add_preloaded_file`. The function expects either a path to a file or a file-like object with the available data (file path has precedence if both are provided). + Common module features ====================== diff --git a/fame/common/email_utils.py b/fame/common/email_utils.py index eaa9cb7..b1934b2 100755 --- a/fame/common/email_utils.py +++ b/fame/common/email_utils.py @@ -68,14 +68,14 @@ class EmailMessage: def __init__(self, server, subject): self.server = server - self.msg = MIMEMultipart('alternative') + self.msg = MIMEMultipart() self.msg['Subject'] = subject self.msg['From'] = server.config.from_address self.msg['Reply-to'] = server.config.replyto or server.config.from_address self.msg['Return-path'] = server.config.replyto or server.config.from_address self.msg.preamble = subject - def add_content(self, text, content_type="text"): + def add_content(self, text, content_type="plain"): self.msg.attach(MIMEText(text, content_type, "utf-8")) def add_attachment(self, filepath, filename=None): @@ -83,11 +83,9 @@ def add_attachment(self, filepath, filename=None): filename = os.path.basename(filepath) with open(filepath, "rb") as f: - self.msg.attach(MIMEApplication( - f.read(), - Content_Disposition='attachment; filename="{}"'.format(filename), - Name=filename - )) + part = MIMEApplication(f.read(), Name=os.path.basename(filepath)) + part['Content-Disposition'] = 'attachment; filename="{0}"'.format(filename) + self.msg.attach(part) def send(self, to, cc=[], bcc=[]): recipients = to + cc + bcc @@ -134,7 +132,7 @@ def __init__(self, template_path=None): def new_message(self, subject, body): msg = EmailMessage(self, subject) - msg.add_content(body, 'text') + msg.add_content(body) return msg diff --git a/fame/core/analysis.py b/fame/core/analysis.py index 698e0b8..a4f2951 100755 --- a/fame/core/analysis.py +++ b/fame/core/analysis.py @@ -1,7 +1,6 @@ import os import datetime import traceback - from shutil import copy from fame.common.config import fame_config @@ -28,6 +27,7 @@ def __init__(self, values): self['pending_modules'] = [] self['waiting_modules'] = [] self['canceled_modules'] = [] + self['preloading_modules'] = [] self['tags'] = [] self['iocs'] = [] self['results'] = {} @@ -48,12 +48,27 @@ def __init__(self, values): if '_id' not in self: self._init_threat_intelligence() + + # Sort preloading and processing modules + if self['modules']: + processing = [] + for module_name in self['modules']: + module = dispatcher.get_module(module_name) + if module is not None: + if module.info['type'] == "Preloading": + self['preloading_modules'].append(module_name) + else: + processing.append(module_name) + + self['modules'] = processing + self.save() if self['modules']: self.queue_modules(self['modules']) - else: - self._automatic() + + self._automatic() + self.resume() # can/will be overridden by the worker implementation def _get_generated_file_path(self, location): @@ -92,13 +107,13 @@ def add_extracted_file(self, filepath, automatic_analysis=True): # Automatically analyze extracted file if magic is enabled and module did not disable it if self.magic_enabled() and automatic_analysis: - modules = None + modules = [] config = Config.get(name="extracted").get_values() if config is not None and "modules" in config: modules = config["modules"].split() f.analyze(self['groups'], self['analyst'], modules, self['options']) - fd.close() + f.add_groups(self['groups']) self.append_to('extracted_files', f['_id']) f.add_parent_analysis(self) @@ -150,6 +165,21 @@ def add_probable_name(self, probable_name): self._file.add_probable_name(probable_name) self.append_to('probable_names', probable_name) + def refresh_iocs(self): + for ioc in self["iocs"]: + value = ioc["value"] + ti_tags, ti_indicators = self._lookup_ioc(value) + if ti_tags: + self.collection.update_one({'_id': self['_id'], 'iocs.value': value}, + {'$set': {'iocs.$.ti_tags': ti_tags}}) + + if ti_indicators: + self.collection.update_one({'_id': self['_id'], 'iocs.value': value}, + {'$set': {'iocs.$.ti_indicators': ti_indicators}}) + + ioc["ti_tags"] = ti_tags + ioc["ti_indicators"] = ti_indicators + def add_ioc(self, value, source, tags=[]): # First, we need to make sure there is a record for this IOC r = self.collection.update_one({'_id': self['_id'], 'iocs.value': {'$ne': value}}, @@ -175,23 +205,49 @@ def add_ioc(self, value, source, tags=[]): self.collection.update_one({'_id': self['_id'], 'iocs.value': value}, {'$addToSet': {'iocs.$.sources': source}}) - # can/will be overridden by the worker implementation - def _store_preloaded_file(self, filepath, fd): - return File(filename=os.path.basename(filepath), stream=fd) + def _store_preloaded_file(self, filepath=None, fd=None): + if not filepath and not fd: + raise ValueError( + "Please provide either the path to the file or a file-like " + "object containing the data.") + + if filepath and fd: + self.log( + "debug", + "Please provide either the path to the file or a " + "file-like object containing the data, not both." + "Choosing the filepath for now.") + + if fame_config.remote: + if filepath: + response = send_file_to_remote(filepath, '/files/') + else: + response = send_file_to_remote(fd, '/files/') + + return File(response.json()['file']) + else: + if filepath: + with open(filepath, 'rb') as f: + return File(filename=os.path.basename(filepath), stream=f) + else: + return File(filename=self._file['names'][0], stream=fd) def add_preloaded_file(self, filepath, fd): f = self._store_preloaded_file(filepath, fd) - f.append_to('analysis', self['_id']) + self._file = f - if f['names'] == ['file']: - f['names'] = self._file['names'] - f.save() + if f['_id'] != self['file']: + f.append_to('analysis', self['_id']) - self['file'] = f['_id'] - self._file = f - self.save() + if f['names'] == ['file']: + f['names'] = self._file['names'] + f.save() + + self['file'] = f['_id'] + self.save() - self._automatic(preloading_done=True) + # Queue general purpose modules if necessary + self._automatic() # Starts / Resumes an analysis to reach the target module def resume(self): @@ -200,21 +256,35 @@ def resume(self): # First, see if there is pending modules remaining if self._run_pending_modules(): was_resumed = True + # If not and there is no file, look for a preloading module + elif self._needs_preloading(): + try: + next_module = dispatcher.next_preloading_module(self['preloading_modules'], self._tried_modules()) + self.queue_modules(next_module) + was_resumed = True + except DispatchingException: + self.log('warning', 'no preloading module was able to find a file for submitted hash') + + for module in list(self['waiting_modules']): + self._cancel_module(module) + # If not, look for a path to a waiting module else: - # If not, look for a path to a waiting module - for module in self['waiting_modules']: + for module in list(self['waiting_modules']): try: next_module = dispatcher.next_module(self._types_available(), module, self._tried_modules()) self.queue_modules(next_module) was_resumed = True except DispatchingException: - self.remove_from('waiting_modules', module) - self.append_to('canceled_modules', module) - self.log('warning', 'could not find execution path to "{}" (cancelled)'.format(module)) + self._cancel_module(module) if not was_resumed and self['status'] != self.STATUS_ERROR: self._mark_as_finished() + def _cancel_module(self, module): + self.remove_from('waiting_modules', module) + self.append_to('canceled_modules', module) + self.log('warning', 'could not find execution path to "{}" (cancelled)'.format(module)) + # Queue execution of specific module(s) def queue_modules(self, modules, fallback_waiting=True): for module_name in iterify(modules): @@ -224,13 +294,16 @@ def queue_modules(self, modules, fallback_waiting=True): module_name not in self['pending_modules']): module = dispatcher.get_module(module_name) - if self._can_execute_module(module): - if self.append_to('pending_modules', module_name): - celery.send_task('run_module', - args=(self['_id'], module_name), - queue=module.info['queue']) - elif fallback_waiting: - self.append_to('waiting_modules', module_name) + if module is None: + self._error_with_module(module_name, "module has been removed or disabled.") + else: + if self._can_execute_module(module): + if self.append_to('pending_modules', module_name): + celery.send_task('run_module', + args=(self['_id'], module_name), + queue=module.info['queue']) + elif fallback_waiting: + self.append_to('waiting_modules', module_name) def add_tag(self, tag): self.append_to('tags', tag) @@ -248,7 +321,10 @@ def filepath(self, path): return path def get_main_file(self): - return self.filepath(self._file['filepath']) + filepath = self._file['filepath'] + if self._needs_preloading(): + return filepath + return self.filepath(filepath) def get_files(self, file_type): results = [] @@ -303,9 +379,16 @@ def _types_available(self): else: return self['generated_files'].keys() + [self._file['type']] + def _needs_preloading(self): + return self._file['type'] == 'hash' + # Determine if a module could be run on the current status of analysis def _can_execute_module(self, module): - if 'acts_on' not in module.info or not module.info['acts_on']: + # Only Preloading modules can execute on a hash + if self._needs_preloading(): + return module.info['type'] == "Preloading" + # When a file is present, look at acts_on property + elif 'acts_on' not in module.info or not module.info['acts_on']: return True else: for source_type in iterify(module.info['acts_on']): @@ -327,24 +410,10 @@ def _tried_modules(self): return self['executed_modules'] + self['canceled_modules'] # Automatic analysis - def _automatic(self, preloading_done=False): - if self.magic_enabled(): - if len(self['pending_modules']) == 0 and self['status'] == self.STATUS_PENDING: - if self._file['type'] == "hash": - self['status'] = self.STATUS_PRELOADING - self.save() - - preloading_module = dispatcher.get_next_preloading_module() - if preloading_module: - self.queue_modules(preloading_module, False) - else: - self.queue_modules(dispatcher.general_purpose(), False) - - if preloading_done and self['status'] == self.STATUS_PRELOADING: - self.queue_modules(dispatcher.general_purpose(), False) - - if len(self['pending_modules']) == 0: - self._mark_as_finished() + def _automatic(self): + # If magic is enabled, schedule general purpose modules + if self.magic_enabled() and not self['modules']: + self.queue_modules(dispatcher.general_purpose(), False) def _error_with_module(self, module, message): self.log("error", "{}: {}".format(module, message)) diff --git a/fame/core/file.py b/fame/core/file.py index 3496ca8..d7252e2 100755 --- a/fame/core/file.py +++ b/fame/core/file.py @@ -10,13 +10,23 @@ from fame.core.module_dispatcher import dispatcher from fame.core.config import Config +from fame.common.email_utils import EmailServer + +notification_body_tpl = u"""Hi, + +{0} has written the following comment on analysis {1}: + +\t{2} + +Best regards""" + def _hash_by_length(hash): _map = { # hashlength: (md5, sha1, sha256) - 32: (hash, "", ""), - 40: ("", hash, ""), - 64: ("", "", hash), + 32: (hash.lower(), "", ""), + 40: ("", hash.lower(), ""), + 64: ("", "", hash.lower()), } return _map.get(len(hash), (None, None, None)) @@ -32,37 +42,6 @@ def __init__(self, values=None, filename=None, stream=None, create=True, self['comments'] = [] MongoDict.__init__(self, values) - elif hash: - MongoDict.__init__(self, {}) - self['probable_names'] = [] - self['parent_analyses'] = [] - self['groups'] = [] - self['owners'] = [] - self['comments'] = [] - - md5, sha1, sha256 = _hash_by_length(hash) - - self.existing = False - - existing_file = ( - ( - # search for existing samples first - self.collection.find_one({'sha256': sha256}) if sha256 else - self.collection.find_one({'sha1': sha1}) if sha1 else - self.collection.find_one({'md5': md5}) if md5 else None - ) - # otherwise, try the hash as filename (aka hash submission) - or self.collection.find_one({'names': [hash]}) - ) - - if existing_file: - self.existing = True - self.update(existing_file) - else: - self._compute_default_properties(hash_only=True) - self._init_hash(hash) - self.save() - else: MongoDict.__init__(self, {}) self['probable_names'] = [] @@ -70,27 +49,66 @@ def __init__(self, values=None, filename=None, stream=None, create=True, self['groups'] = [] self['owners'] = [] self['comments'] = [] + self['analysis'] = [] + if hash: + self._init_with_hash(hash) + else: + self._init_with_file(filename, stream, create) - # filename should be set - if filename is not None and stream is not None: - self._compute_hashes(stream) + def _init_with_hash(self, hash): + md5, sha1, sha256 = _hash_by_length(hash) - # If the file already exists in the database, update it - self.existing = False - existing_file = self.collection.find_one({'sha256': self['sha256']}) + self.existing = False - if existing_file: - self._add_to_previous(existing_file, filename) - self.existing = True + # Set hash and look for existing files + existing_file = None - # Otherwise, compute default properties and save - elif create: - self._store_file(filename, stream) - self._compute_default_properties() - self.save() + if sha256: + self['sha256'] = sha256 + existing_file = self.collection.find_one({'sha256': sha256}) + elif sha1: + self['sha1'] = sha1 + existing_file = self.collection.find_one({'sha1': sha1}) + elif md5: + self['md5'] = md5 + existing_file = self.collection.find_one({'md5': md5}) + else: + # otherwise, try the hash as filename (aka hash submission) + self.collection.find_one({'names': [hash]}) - def add_comment(self, analyst_id, comment, analysis_id=None, probable_name=None): + if existing_file: + self.existing = True + self.update(existing_file) + else: + self._compute_default_properties(hash_only=True) + self._init_hash(hash) + self.save() + + def _init_with_file(self, filename, stream, create): + # filename should be set + if filename is not None and stream is not None: + self._compute_hashes(stream) + + # If the file already exists in the database, update it + self.existing = False + existing_file = ( + self.collection.find_one({'sha256': self['sha256']}) or + self.collection.find_one({'sha1': self['sha1']}) or + self.collection.find_one({'md5': self['md5']}) + ) + + if existing_file: + self._add_to_previous(existing_file, filename) + self.existing = True + + # If the file doesn't exist, or exists as a hash submission, compute default properties and save + if create and ((existing_file is None) or (self['type'] == 'hash')): + self._store_file(filename, stream) + self._compute_default_properties() + self.save() + + def add_comment(self, analyst_id, comment, analysis_id=None, probable_name=None, notify=None): if probable_name: self.add_probable_name(probable_name) @@ -101,6 +119,33 @@ def add_comment(self, analyst_id, comment, analysis_id=None, probable_name=None) 'probable_name': probable_name, 'date': datetime.datetime.now() }) + if notify is not None and analysis_id is not None: + self.notify_new_comment(analysis_id, analyst_id, comment) + + def notify_new_comment(self, analysis_id, commentator_id, comment): + commentator = store.users.find_one({'_id': commentator_id}) + analysis = store.analysis.find_one({'_id': ObjectId(analysis_id)}) + analyst_id = analysis['analyst'] + recipients = set() + # First let's add submiter analyst and check if he is not commentator + if commentator_id != analyst_id: + analyst = store.users.find_one({'_id': analysis['analyst']}) + recipients.add(analyst['email']) + # iter on commentators and add them as recipient + for comment in self['comments']: + if comment['analyst'] not in [analyst_id, commentator_id]: + recipient = store.users.find_one({'_id': comment['analyst']}) + recipients.add(recipient['email']) + if len(recipients): + config = Config.get(name="email").get_values() + analysis_url = "{0}/analyses/{1}".format(fame_config.fame_url, analysis_id) + body = notification_body_tpl.format(commentator['name'], + analysis_url, + comment['comment']) + email_server = EmailServer() + if email_server.is_connected: + msg = email_server.new_message("[FAME] New comment on analysis", body) + msg.send(list(recipients)) def add_probable_name(self, probable_name): for name in self['probable_names']: @@ -115,6 +160,9 @@ def add_owners(self, owners): def remove_group(self, group): # Update file + self.remove_from('groups', group) + + # Update previous analysis for analysis_id in self['analysis']: analysis = Analysis(store.analysis.find_one({'_id': ObjectId(analysis_id)})) analysis.remove_from('groups', group) diff --git a/fame/core/module.py b/fame/core/module.py index 2e11f65..70e6894 100755 --- a/fame/core/module.py +++ b/fame/core/module.py @@ -50,22 +50,15 @@ def details_template(self): return '/'.join(self['path'].split('.')[2:-1]) + '/details.html' def update_config(self, new_info): - def _update_queue(): - self['queue'] = new_info['queue'] - if 'queue' in self['diffs']: - if self['diffs']['queue'] == new_info['queue']: - del self['diffs']['queue'] - else: - self['queue'] = self['diffs']['queue'] - if self['type'] == 'Processing': self['generates'] = new_info['generates'] - _update_queue() + self._update_diffed_value('queue', new_info['queue']) self._update_diffed_value('acts_on', new_info['acts_on']) self._update_diffed_value('triggered_by', new_info['triggered_by']) elif self['type'] == 'Preloading': - _update_queue() + self._update_diffed_value('queue', new_info['queue']) + self._update_diffed_value('priority', new_info['priority']) elif self['type'] == 'Filetype': self._update_diffed_value('acts_on', new_info['acts_on']) @@ -124,25 +117,33 @@ def _remove_value(self, name, value): self['diffs'][name]['removed'].append(value) def _update_diffed_value(self, name, value): - self._init_list_diff(name) - self[name] = copy(value) + if is_iterable(value): + self._init_list_diff(name) + self[name] = value - if name in self['diffs']: - new_removed = [] - for element in self['diffs'][name]['removed']: - if element in self[name]: - self[name].remove(element) - new_removed.append(element) + if name in self['diffs']: + new_removed = [] + for element in self['diffs'][name]['removed']: + if element in self[name]: + self[name].remove(element) + new_removed.append(element) - self['diffs'][name]['removed'] = new_removed + self['diffs'][name]['removed'] = new_removed - new_added = [] - for element in self['diffs'][name]['added']: - if element not in self[name]: - self[name].append(element) - new_added.append(element) + new_added = [] + for element in self['diffs'][name]['added']: + if element not in self[name]: + self[name].append(element) + new_added.append(element) - self['diffs'][name]['added'] = new_added + self['diffs'][name]['added'] = new_added + else: + self[name] = value + if name in self['diffs']: + if self['diffs'][name] == value: + del self['diffs'][name] + else: + self[name] = self['diffs'][name] class Module(object): @@ -256,6 +257,15 @@ def init_config(self): if config['value'] is None: setattr(self, config['name'], config['default']) + def log(self, level, message): + """Add a log message to the analysis + + Args: + level: string to define the log level (``debug``, ``info``, ``warning`` or ``error``). + message: free text message containing the log information. + """ + self._analysis.log(level, "%s: %s" % (self.name, message)) + @classmethod def named_config(cls, name): config = { @@ -316,15 +326,6 @@ def __init__(self, with_config=True): self.results = None self.tags = [] - def log(self, level, message): - """Add a log message to the analysis - - Args: - level: string to define the log level (``debug``, ``info``, ``warning`` or ``error``). - message: free text message containing the log information. - """ - self._analysis.log(level, "%s: %s" % (self.name, message)) - def register_files(self, file_type, locations): """Add a generated file to the analysis. @@ -974,7 +975,7 @@ class FiletypeModule(Module): acts_on = [] - def recognize(filepath, current_type): + def recognize(self, filepath, current_type): """To implement. Checks the file in order to determine more accurate type. @@ -1116,16 +1117,27 @@ def static_info(cls): class PreloadingModule(Module): - """ PreloadingModules can be used to download the sample - binary from e.g. VirusTotal before queueing any - processing modules. Hence, PreloadingModules only work - on hashes. A successful execution of a PreloadingModule - updates the Analysis object with the new data and queues - the remaining modules as if the sample itself was uploaded - the FAME. + """Base class for preloading modules + + PreloadingModules can be used to download the sample + binary from e.g. VirusTotal before queueing any + processing modules. Hence, PreloadingModules only work + on hashes. A successful execution of a PreloadingModule + updates the Analysis object with the new data and queues + the remaining modules as if the sample itself was uploaded + the FAME. + + Attributes: + queue: A string defining on which queue the tasks will be added. This + defines on which worker this module will execute. The default + value is `unix`. + + priority: An integer defining the module's priority when preloading. + The smallest values are used first (defaults to 100). """ queue = 'unix' + priority = 100 def __init__(self, with_config=True): Module.__init__(self, with_config) @@ -1137,13 +1149,13 @@ def preload(self, target): Args: target (string): the hash that is to be analyzed - Raises: - ModuleExecutionError: Preloading the analysis failed (e.g. - no file for a given hash was found). + Returns: + A boolean indicating whether or not a file could be downloaded + for the given hash """ raise NotImplementedError - def add_preloaded_file(self, filepath, fd): + def add_preloaded_file(self, filepath=None, fd=None): self._analysis.add_preloaded_file(filepath, fd) def init_options(self, options): @@ -1151,18 +1163,19 @@ def init_options(self, options): setattr(self, option, options[option]) def execute(self, analysis): - self._analysis = analysis - self.init_options(analysis['options']) - return self.preload(self._analysis.get_main_file()) - - def log(self, level, message): - """Add a log message to the analysis - - Args: - level: string to define the log level (``debug``, ``info``, ``warning`` or ``error``). - message: free text message containing the log information. - """ - self._analysis.log(level, "%s: %s" % (self.name, message)) + try: + self._analysis = analysis + self.init_options(analysis['options']) + return self.preload(self._analysis.get_main_file()) + except ModuleExecutionError, e: + self.log("error", "Could not run on %s: %s" % ( + self._analysis.get_main_file(), e)) + return False + except: + tb = traceback.format_exc() + self.log("error", "Exception occurred while execting module on %s.\n %s" % ( + self._analysis.get_main_file(), tb)) + return False @classmethod def static_info(cls): @@ -1172,7 +1185,8 @@ def static_info(cls): "type": "Preloading", "config": cls.config, "diffs": {}, - "queue": cls.queue + "queue": cls.queue, + "priority": cls.priority } init_config_values(info) diff --git a/fame/core/module_dispatcher.py b/fame/core/module_dispatcher.py index f7635b1..a1c985e 100755 --- a/fame/core/module_dispatcher.py +++ b/fame/core/module_dispatcher.py @@ -98,17 +98,25 @@ def next_module(self, types_available, module_name, excluded_modules): else: return self._shortest_path_to_module(types_available, module, excluded_modules) - def get_next_preloading_module(self, excluded_modules=[]): - candidate_modules = [] + def next_preloading_module(self, selected_modules=[], excluded_modules=[]): + candidate_module = None + smallest_priority = None for module in self.get_preloading_modules(): - if module.info['name'] not in excluded_modules: - candidate_modules.append(module.info['name']) - - if len(candidate_modules) > 0: - return candidate_modules[0] + if ( + (not selected_modules or module.info['name'] in selected_modules) + and module.info['name'] not in excluded_modules + ): + module_info = ModuleInfo.get(name=module.info['name']) + + if smallest_priority is None or module_info['priority'] < smallest_priority: + candidate_module = module_info['name'] + smallest_priority = module_info['priority'] + + if candidate_module: + return candidate_module else: - return "" + raise DispatchingException("No more preloading module available") # Get all generale purpose modules def general_purpose(self): diff --git a/fame/worker/analysis.py b/fame/worker/analysis.py index daab544..93b6147 100755 --- a/fame/worker/analysis.py +++ b/fame/worker/analysis.py @@ -93,13 +93,6 @@ def run(self, module_name): self.add_tag(module_name) - elif module.info['type'] == "Preloading": - # queue next preloading module - next_module = dispatcher.get_next_preloading_module( - self._tried_modules()) - if next_module: - self.queue_modules(next_module) - self.log('debug', "Done with {0}".format(module_name)) except Exception: tb = traceback.format_exc() diff --git a/utils/initial_data.py b/utils/initial_data.py index 1e21f2d..7ff3068 100755 --- a/utils/initial_data.py +++ b/utils/initial_data.py @@ -36,6 +36,9 @@ def create_types(): message/rfc822 = eml application/CDFV2-unknown = msg application/java-archive = jar +application/x-7z-compressed = 7z +application/x-rar = rar +application/x-iso9660-image = iso [details] @@ -63,7 +66,11 @@ def create_types(): jar = jar zip = zip msg = msg -eml = eml""", +eml = eml +iso = iso +msi = executable +7z = 7z +rar = rar""", 'description': "In order to determine the file type, FAME will use the `python-magic` library. It will then try to find a match in 'mappings' for either the extension, the detailed type or the mime type (in this order of priority). If no matching type was found, the mime type will be used." } ] diff --git a/web/auth/ad/REAMDE.md b/web/auth/ad/REAMDE.md new file mode 100644 index 0000000..ce940b1 --- /dev/null +++ b/web/auth/ad/REAMDE.md @@ -0,0 +1,28 @@ +# Installation + +*This module requires a Windows AD server and is not compatible with OpenLDAP servers.* + +The AD authentication module requires the python-ldap package to be available to FAME. It can be installed with this command: `utils/run.sh -m pip install python-ldap~=3.2.0`. This installs the required package into the virtualenv that is used by FAME. + +Once installed, please change the authentication type of fame to "ad" if you want to use the AD authenticator. + +# Configuration + +This authenticator requires several config values to set prior to using the authenticator. The settings are shown in the following table. These settings need to be put into the fame config file. + +| Name | Description| +|------|------------| +| ldap_uri | The LDAP URI of the LDAP server. Example: `ldap://dc.example.com` | +| ldap_user | The user that is used to access the LDAP server. | +| ldap_password | The password for the LDAP user. | +| ldap_filter_email | The LDAP filter query that selects user objects. Example query to select user objects by their email address: `(&(objectCategory=Person)(sAMAccountName=*)(mail={}))` | +| ldap_filter_dn | The LDAP filter for the DN. Example: `OU=People,DC=example,DC=com`| + + +# Custom CA files + +python-ldap uses the system certificate storage to check for CA files. Thus, if you have a custom or self-signed CA you need to install it in the system and update the certificate store. Instructions for Ubuntu are: +```bash +cp /usr/local/share/ca-certificates/ +update-ca-certificates +``` diff --git a/web/auth/ad/__init__.py b/web/auth/ad/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/web/auth/ad/config/__init__.py b/web/auth/ad/config/__init__.py new file mode 100644 index 0000000..ce17c46 --- /dev/null +++ b/web/auth/ad/config/__init__.py @@ -0,0 +1,6 @@ + + +try: + from .custom_mappings import * +except ImportError: + pass diff --git a/web/auth/ad/config/custom_mappings.py.sample b/web/auth/ad/config/custom_mappings.py.sample new file mode 100755 index 0000000..f7f24b4 --- /dev/null +++ b/web/auth/ad/config/custom_mappings.py.sample @@ -0,0 +1,17 @@ +ROLE_MAPPING = { + "GROUP_NAME": { + "permissions": [ + "submit_iocs", + "configs", + "cuckoo_access", + ], + "default_sharing": [ + "cert", + "users" + ], + "groups": [ + "cert", + "*", + ], + } +} diff --git a/web/auth/ad/templates/base_unauthenticated.html b/web/auth/ad/templates/base_unauthenticated.html new file mode 100755 index 0000000..c98b703 --- /dev/null +++ b/web/auth/ad/templates/base_unauthenticated.html @@ -0,0 +1,79 @@ + + + + + + + + FAME - {% block title %} {% endblock %} + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + +{% with messages = get_flashed_messages(with_categories=true) %} + {% if messages %} + + {% endif %} +{% endwith %} + + + + + + diff --git a/web/auth/ad/templates/login.html b/web/auth/ad/templates/login.html new file mode 100755 index 0000000..d34cc5a --- /dev/null +++ b/web/auth/ad/templates/login.html @@ -0,0 +1,25 @@ +{% extends "base_unauthenticated.html" %} + +{% block title %}Login{% endblock %} + +{% block body %} +
+ +
+
+
+ + +
+
+ + +
+
+
+ +
+
+ +
+{% endblock %} diff --git a/web/auth/ad/user_management.py b/web/auth/ad/user_management.py new file mode 100755 index 0000000..868f2dd --- /dev/null +++ b/web/auth/ad/user_management.py @@ -0,0 +1,185 @@ +import codecs +import os +import ldap +from itsdangerous import TimestampSigner +from flask_login import login_user, make_secure_token + +from fame.core.user import User +from fame.common.config import fame_config +from web.auth.ad.config import ROLE_MAPPING +from web.views.helpers import user_if_enabled + +ldap.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_X_TLS_DEMAND) + + +class LdapSettingsNotPresentException(Exception): + pass + + +def _check_ldap_settings_present(): + def _check(name): + if name not in fame_config: + print name + " not present in config" + return False + return True + + return ( + _check("ldap_uri") and _check("ldap_user") and + _check("ldap_password") and _check("ldap_filter_dn") and + _check("ldap_filter_email") + ) + + +def _ldap_get_con(): + if not _check_ldap_settings_present(): + return None + + con = ldap.initialize(fame_config.ldap_uri) + con.protocol_version = ldap.VERSION3 + con.set_option(ldap.OPT_REFERRALS, 0) + return con + + +def _find_user_by_email(con, email): + try: + con.simple_bind_s(fame_config.ldap_user, fame_config.ldap_password) + except ldap.INVALID_CREDENTIALS: + print "Cannot connect to LDAP: invalid credentials" + return None + + users = con.search_s( + fame_config.ldap_filter_dn, ldap.SCOPE_SUBTREE, + filterstr=fame_config.ldap_filter_email.format(email) + ) + + ldap_user = None + + if users: + user = users[0][1] + + principal = None + if 'userPrincipalName' in user and len(user['userPrincipalName']) != 0: + principal = user['userPrincipalName'][0].decode() + + full_name = user['cn'][0].decode() + email = user['mail'][0].decode() + enabled = (int(user['userAccountControl'][0].decode()) & 0x2) == 0 + groups = [group for group in map( + lambda x: x.decode().lower().split(",")[0].lstrip("cn="), + user['memberOf'] + )] + + ldap_user = { + "principal": principal or full_name, + "name": full_name, + "mail": email, + "enabled": enabled, + "groups": groups + } + + return ldap_user + + +def ldap_authenticate(email, password): + con = _ldap_get_con() + if not con: + raise LdapSettingsNotPresentException + + ldap_user = _find_user_by_email(con, email) + + if ldap_user: + try: + con.simple_bind_s(ldap_user['principal'], password) + return ldap_user + except ldap.INVALID_CREDENTIALS: + # forward exception to view + raise + finally: + con.unbind_s() + + +def auth_token(user): + return codecs.encode(os.urandom(12), 'hex').decode() + make_secure_token(user['email'], os.urandom(32)) + + +def password_reset_token(user): + signer = TimestampSigner(fame_config.secret_key) + + return signer.sign(str(user['_id'])) + + +def validate_password_reset_token(token): + signer = TimestampSigner(fame_config.secret_key) + + return signer.unsign(token, max_age=86400).decode() + + +def get_mapping(collection, name): + result = set() + for source_group in collection: + for mapping in ROLE_MAPPING.get(source_group, {}).get(name, []): + result.update(mapping) + return list(result) + + +def create_user(ldap_user): + groups = get_mapping(ldap_user['groups'], "groups") + default_sharing = get_mapping(ldap_user['groups'], "default_sharing") + permissions = get_mapping(ldap_user["groups"], "permissions") + + user = User({ + 'name': ldap_user['name'], + 'email': ldap_user['mail'], + 'enabled': ldap_user['enabled'], + 'groups': groups, + 'default_sharing': default_sharing, + 'permissions': permissions, + }) + user.save() + user.generate_avatar() + + return user + + +def update_or_create_user(ldap_user): + user = User.get(email=ldap_user['mail']) + + if user: + # update groups + groups = get_mapping(ldap_user['groups'], "groups") + user.update_value('groups', groups) + + # update default sharings + default_sharing = get_mapping(ldap_user['groups'], "default_sharing") + user.update_value('default_sharing', default_sharing) + + # update permissions + permissions = get_mapping(ldap_user["groups"], "permissions") + user.update_value('permissions', permissions) + + # enable/disable user + user.update_value('enabled', ldap_user['enabled']) + + return user_if_enabled(user) + + return create_user(ldap_user) + + +def authenticate(email, password): + ldap_user = ldap_authenticate(email, password) + + if not ldap_user: + # user not found in LDAP, update local user object accordingly (if existent) + user = User.get(email=email) + if user: + print "Disabling user {}: not available in LDAP".format(email) + user.update_value('enabled', False) + + return user + + user = update_or_create_user(ldap_user) + + if user: + login_user(user) + + return user diff --git a/web/auth/ad/views.py b/web/auth/ad/views.py new file mode 100755 index 0000000..d2d4d23 --- /dev/null +++ b/web/auth/ad/views.py @@ -0,0 +1,42 @@ +from flask import Blueprint, render_template, request, redirect, flash +from flask_login import logout_user + +from web.views.helpers import prevent_csrf, user_has_groups_and_sharing +from web.auth.ad.user_management import authenticate, LdapSettingsNotPresentException + +from ldap import SERVER_DOWN, INVALID_CREDENTIALS + + +auth = Blueprint('auth', __name__, template_folder='templates') + + +@auth.route('/login', methods=['GET', 'POST']) +@prevent_csrf +def login(): + if request.method == 'GET': + return render_template('login.html') + else: + try: + user = authenticate(request.form.get('email'), request.form.get('password')) + except SERVER_DOWN: + flash("LDAP Server down.", "danger") + return render_template('login.html') + except INVALID_CREDENTIALS: + flash("Invalid credentials.", "danger") + return render_template('login.html') + except LdapSettingsNotPresentException: + flash("LDAP Settings not present. Check server logs.", "danger") + return render_template('login.html') + + if not user or not user_has_groups_and_sharing(user): + flash("Access not allowed.", "danger") + return render_template('login.html') + + redir = request.args.get('next', '/') + return redirect(redir) + + +@auth.route('/logout') +def logout(): + logout_user() + return redirect('/login') diff --git a/web/auth/single_user/__init__.py b/web/auth/single_user/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/web/auth/single_user/views.py b/web/auth/single_user/views.py new file mode 100755 index 0000000..15eb0b0 --- /dev/null +++ b/web/auth/single_user/views.py @@ -0,0 +1,45 @@ +from flask_login import login_user +from flask import Blueprint, request, redirect + +from fame.core.user import User +from web.views.helpers import prevent_csrf + + +auth = Blueprint('auth', __name__, template_folder='templates') + + +def get_or_create_user(): + user = User.get(email="admin@fame") + + if not user: + user = User({ + 'name': "admin", + 'email': "admin@fame", + 'groups': ['admin', '*'], + 'default_sharing': ['admin'], + 'permissions': ['*'], + 'enabled': True + }) + user.save() + user.generate_avatar() + + return user + + +@auth.route('/login', methods=['GET', 'POST']) +@prevent_csrf +def login(): + redir = request.args.get('next', '/') + + if "/login" in redir: + redir = '/' + + login_user(get_or_create_user()) + + return redirect(redir) + + +@auth.route('/logout') +def logout(): + redir = '/' + return redirect(redir) diff --git a/web/templates/analyses/show.html b/web/templates/analyses/show.html index 1571e98..0464904 100755 --- a/web/templates/analyses/show.html +++ b/web/templates/analyses/show.html @@ -267,6 +267,9 @@

Observables

{% endif %} {% endfor %} {% endif %} +
  • + Refresh Observables +
  • Fame diff --git a/web/templates/files/details.html b/web/templates/files/details.html index d535049..d2ae133 100755 --- a/web/templates/files/details.html +++ b/web/templates/files/details.html @@ -80,25 +80,25 @@

    Object Details


    - {% if file.type != 'url' %}
      - {% if file.type != 'hash' %} -
    • Download
      • - {% endif %} {% if request.endpoint == 'AnalysesView:get' %}
    • All analyses
      • {% endif %} + {% if file.type != 'url' and file.type != 'hash' %} + +
    • Download
      • - {% for module in av_modules %} + {% for module in av_modules %}
    • {{module}}
    • Send to {{module}}
      • - {% endfor %} + {% endfor %} + {% endif %}
    - {% endif %}
    @@ -143,7 +142,7 @@

    Comments

    - + {% if analysis %} {% endif %} @@ -156,16 +155,30 @@

    Comments

    {% endif %} - +
    - +
    + + + + +
    +
    - +
    diff --git a/web/templates/modules/index.html b/web/templates/modules/index.html index 798b73c..c6ec94c 100755 --- a/web/templates/modules/index.html +++ b/web/templates/modules/index.html @@ -236,6 +236,69 @@

    Error

    + +
    +
    +
    +
    +

    Preloading Modules

    +

    +
    +
    + {% for module in data.modules['Preloading'] %} +
    +
    +
    +
    {{module.name}}
    +
    +
    + Configure + {% if module.enabled %} +
    + +
    + {% else %} + {% if not module.error %} +
    + +
    + {% endif %} + {% endif %} +
    +
    + {% if module.description %} +
    {{module.description}}
    + {% endif %} + + {% if module.error %} +
    +
    +

    Error

    +
    +
    + 
    {{module.error}}
    +
    +
    + {% endif %} + +
      +
    • Priority {{module.priority}}
      • +
    • Queue {{module.queue}}
      • + {% if module.enabled %} +
    • enabled
      • + {% else %} +
    • Disabled
      • + {% endif %} +
    +
    + {% else %} +
    No module to list. Consider adding a module repository.
    + {% endfor %} +
    +
    +
    +
    +
    diff --git a/web/templates/modules/module_configuration.html b/web/templates/modules/module_configuration.html index b4979dd..d99b354 100755 --- a/web/templates/modules/module_configuration.html +++ b/web/templates/modules/module_configuration.html @@ -23,10 +23,10 @@

    {{data.module.name}}

    - +
    - -

    Comma-delimited list of types that this module is able to handle (ex: word,excel).

    + +

    The module's priority when preloading. The smallest values are used first (defaults to 100).

    diff --git a/web/views/analyses.py b/web/views/analyses.py index 244642b..9462dd3 100755 --- a/web/views/analyses.py +++ b/web/views/analyses.py @@ -256,7 +256,7 @@ def post(self): options = get_options() if options is None: return validation_error() - + valid_submission = self._validate_form(groups, modules, options) if not valid_submission: return validation_error() @@ -386,3 +386,16 @@ def download_support_file(self, id, module, filename): return file_download(filepath) else: abort(404) + + @route('//refresh-iocs') + def refresh_iocs(self, id): + """Refresh IOCs with Threat Intel modules + + .. :quickref: Analysis; Refresh IOCs with Threat Intel modules. + + :param id: id of the analysis. + """ + analysis = Analysis(get_or_404(current_user.analyses, _id=id)) + analysis.refresh_iocs() + + return redirect(analysis, url_for('AnalysesView:get', id=analysis["_id"])) diff --git a/web/views/files.py b/web/views/files.py index 558aa12..8c34161 100755 --- a/web/views/files.py +++ b/web/views/files.py @@ -16,7 +16,6 @@ ) from web.views.mixins import UIView - def return_file(file): analyses = list(current_user.analyses.find({'_id': {'$in': file['file']['analysis']}})) file['av_modules'] = [m.name for m in dispatcher.get_antivirus_modules()] @@ -206,13 +205,14 @@ def add_comment(self, id): comment = request.form.get('comment') analysis_id = request.form.get('analysis') + notify = request.form.get('notify') if comment: # If there is an analysis ID, make sure it is accessible if analysis_id: get_or_404(current_user.analyses, _id=analysis_id) - f.add_comment(current_user['_id'], comment, analysis_id, probable_name) + f.add_comment(current_user['_id'], comment, analysis_id, probable_name, notify) else: flash('Comment should not be empty', 'danger') diff --git a/web/views/helpers.py b/web/views/helpers.py index 3d290a7..b348c4e 100755 --- a/web/views/helpers.py +++ b/web/views/helpers.py @@ -85,6 +85,12 @@ def clean_repositories(repositories): return repositories +def user_has_groups_and_sharing(user): + if len(user['groups']) > 0 and len(user['default_sharing']) > 0: + return True + return False + + def user_if_enabled(user): if user and user['enabled']: return user diff --git a/web/views/modules.py b/web/views/modules.py index a62db11..a53886e 100755 --- a/web/views/modules.py +++ b/web/views/modules.py @@ -3,7 +3,7 @@ from shutil import move, rmtree from time import time from zipfile import ZipFile -from flask import url_for, request, flash, make_response +from flask import url_for, request, flash from flask_classy import FlaskView, route from uuid import uuid4 from markdown2 import markdown @@ -85,6 +85,29 @@ def update_config(settings, options=False): return None +def update_queue(module, new_queue): + if new_queue == '': + flash('queue cannot be empty', 'danger') + return validation_error() + elif module['queue'] != new_queue: + module.update_setting_value('queue', new_queue) + updates = Internals(get_or_404(Internals.get_collection(), name="updates")) + updates.update_value("last_update", time()) + + flash('Workers will reload once they are done with their current tasks', 'success') + + +def update_priority(module, new_priority): + if not new_priority: + new_priority = '100' + + try: + module.update_setting_value('priority', int(new_priority)) + except ValueError: + flash('priority must be an integer', 'danger') + return validation_error() + + class ModulesView(FlaskView, UIView): @requires_permission('manage_modules') @@ -355,11 +378,14 @@ def update_queue(): module.update_setting_value('triggered_by', request.form.get('triggered_by', '')) if 'queue' in request.form: - update_queue() + update_queue(module, request.form.get('queue', '')) elif module['type'] == "Preloading": if 'queue' in request.form: - update_queue() + update_queue(module, request.form.get('queue', '')) + + if 'priority' in request.form: + update_priority(module, request.form.get('priority', '')) errors = update_config(module['config'], options=(module['type'] in ['Preloading', 'Processing'])) if errors is not None: diff --git a/webserver.py b/webserver.py index c153297..a3ca8c6 100755 --- a/webserver.py +++ b/webserver.py @@ -7,7 +7,7 @@ from datetime import datetime from flask import Flask, redirect, request, url_for from flask_login import LoginManager -from werkzeug import url_encode +from werkzeug.urls import url_encode from importlib import import_module from fame.core import fame_init