From e3271a4d6d34e8c3067cfeb204ac7e45a71ac788 Mon Sep 17 00:00:00 2001 From: jnettesheim Date: Thu, 28 Jun 2018 11:06:46 -0700 Subject: [PATCH] Added artifacts filter support for files and Windows Registry #1313 --- plaso/cli/extraction_tool.py | 1 + plaso/cli/helpers/__init__.py | 1 + plaso/cli/helpers/artifact_definitions.py | 29 ++++- plaso/cli/helpers/artifact_filters.py | 96 ++++++++++++++++ plaso/cli/image_export_tool.py | 37 ++++--- plaso/cli/log2timeline_tool.py | 32 +++--- plaso/cli/pinfo_tool.py | 2 + plaso/cli/psteal_tool.py | 27 +++-- plaso/cli/status_view.py | 10 +- plaso/cli/storage_media_tool.py | 1 + plaso/containers/sessions.py | 8 ++ plaso/engine/artifact_filters.py | 30 +++-- plaso/engine/configurations.py | 4 + plaso/engine/engine.py | 122 +++++++++++++++++++-- plaso/parsers/winreg.py | 87 ++++++++++++--- test_data/artifacts/artifact_names | 3 + test_data/artifacts/artifacts_filters.yaml | 5 +- tests/cli/helpers/artifact_definitions.py | 8 +- tests/cli/helpers/artifact_filters.py | 102 +++++++++++++++++ tests/cli/image_export_tool.py | 29 +++++ tests/cli/pinfo_tool.py | 1 + tests/cli/psteal_tool.py | 1 + tests/engine/artifact_filters.py | 10 +- tests/parsers/test_lib.py | 5 +- tests/parsers/winreg.py | 65 +++++++++++ 25 files changed, 630 insertions(+), 86 deletions(-) create mode 100644 plaso/cli/helpers/artifact_filters.py create mode 100644 test_data/artifacts/artifact_names create mode 100644 tests/cli/helpers/artifact_filters.py diff --git a/plaso/cli/extraction_tool.py b/plaso/cli/extraction_tool.py index 3ad1fc1e8f..f1efa24cae 100644 --- a/plaso/cli/extraction_tool.py +++ b/plaso/cli/extraction_tool.py @@ -75,6 +75,7 @@ def _CreateProcessingConfiguration(self, knowledge_base): """ # TODO: pass preferred_encoding. configuration = configurations.ProcessingConfiguration() + configuration.artifact_filters = self._artifact_filters configuration.credentials = self._credential_configurations configuration.debug_output = self._debug_mode configuration.event_extraction.text_prepend = self._text_prepend diff --git a/plaso/cli/helpers/__init__.py b/plaso/cli/helpers/__init__.py index 7de9cb6ed7..f9bd2e5726 100644 --- a/plaso/cli/helpers/__init__.py +++ b/plaso/cli/helpers/__init__.py @@ -3,6 +3,7 @@ from plaso.cli.helpers import analysis_plugins from plaso.cli.helpers import artifact_definitions +from plaso.cli.helpers import artifact_filters from plaso.cli.helpers import data_location from plaso.cli.helpers import date_filters from plaso.cli.helpers import dynamic_output diff --git a/plaso/cli/helpers/artifact_definitions.py b/plaso/cli/helpers/artifact_definitions.py index 9381dbde86..2141110be9 100644 --- a/plaso/cli/helpers/artifact_definitions.py +++ b/plaso/cli/helpers/artifact_definitions.py @@ -43,6 +43,15 @@ def AddArguments(cls, argument_group): 'quickly collect data of interest, such as specific files or ' 'Windows Registry keys.')) + argument_group.add_argument( + '--custom_artifact_definitions', '--custom-artifact-definitions', + dest='custom_artifact_definitions_path', type=str, metavar='PATH', + action='store', help=( + 'Path to a file containing custom artifact definitions, which are ' + '.yaml files. Artifact definitions can be used to describe and ' + 'quickly collect data of interest, such as specific files or ' + 'Windows Registry keys.')) + @classmethod def ParseOptions(cls, options, configuration_object): """Parses and validates options. @@ -86,6 +95,13 @@ def ParseOptions(cls, options, configuration_object): raise errors.BadConfigOption( 'Unable to determine path to artifact definitions.') + custom_artifacts_path = getattr( + options, 'custom_artifact_definitions_path', None) + + if custom_artifacts_path and not os.path.isfile(custom_artifacts_path): + raise errors.BadConfigOption( + 'No such artifacts filter file: {0:s}.'.format(custom_artifacts_path)) + registry = artifacts_registry.ArtifactDefinitionsRegistry() reader = artifacts_reader.YamlArtifactsReader() @@ -102,7 +118,18 @@ def ParseOptions(cls, options, configuration_object): raise errors.BadConfigOption( 'Missing required artifact definition: {0:s}'.format(name)) - setattr(configuration_object, '_artifacts_registry', registry) + if custom_artifacts_path: + try: + registry.ReadFromFile(reader, custom_artifacts_path) + + except (KeyError, artifacts_errors.FormatError) as exception: + raise errors.BadConfigOption(( + 'Unable to read artifact definitions from: {0:s} with error: ' + '{1!s}').format(custom_artifacts_path, exception)) + + setattr(configuration_object, '_artifact_definitions_path', artifacts_path) + setattr( + configuration_object, '_custom_artifacts_path', custom_artifacts_path) manager.ArgumentHelperManager.RegisterHelper(ArtifactDefinitionsArgumentsHelper) diff --git a/plaso/cli/helpers/artifact_filters.py b/plaso/cli/helpers/artifact_filters.py new file mode 100644 index 0000000000..2801f23221 --- /dev/null +++ b/plaso/cli/helpers/artifact_filters.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +"""The artifacts filter file CLI arguments helper.""" + +from __future__ import unicode_literals + +import os + +from plaso.cli import tools +from plaso.cli.helpers import interface +from plaso.cli.helpers import manager +from plaso.lib import errors + + +class ArtifactFiltersArgumentsHelper(interface.ArgumentsHelper): + """Artifacts filter file CLI arguments helper.""" + + NAME = 'artifact_filters' + DESCRIPTION = 'Artifact filters command line arguments.' + + @classmethod + def AddArguments(cls, argument_group): + """Adds command line arguments to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group (argparse._ArgumentGroup|argparse.ArgumentParser): + argparse group. + """ + argument_group.add_argument( + '--artifact_filters', '--artifact-filters', + dest='artifact_filters', type=str, default=None, + action='store', help=( + 'Names of forensic artifact definitions, provided on the command ' + 'command line (comma separated). Forensic artifacts are stored ' + 'in .yaml files that are directly pulled from the artifact ' + 'definitions project. You can also specify a custom ' + 'artifacts yaml file (see --custom_artifact_definitions). Artifact ' + 'definitions can be used to describe and quickly collect data of ' + 'interest, such as specific files or Windows Registry keys.')) + + argument_group.add_argument( + '--artifact_filters_file', '--artifact-filters_file', + dest='artifact_filters', type=str, default=None, + action='store', help=( + 'Names of forensic artifact definitions, provided in a file with ' + 'one artifact name per line. Forensic artifacts are stored in ' + '.yaml files that are directly pulled from the artifact ' + 'definitions project. You can also specify a custom artifacts ' + 'yaml file (see --custom_artifact_definitions). Artifact ' + 'definitions can be used to describe and quickly collect data of ' + 'interest, such as specific files or Windows Registry keys.')) + + @classmethod + def ParseOptions(cls, options, configuration_object): + """Parses and validates options. + + Args: + options (argparse.Namespace): parser options. + configuration_object (CLITool): object to be configured by the argument + helper. + + Raises: + BadConfigObject: when the configuration object is of the wrong type. + BadConfigOption: if the required artifact definitions are not defined. + """ + if not isinstance(configuration_object, tools.CLITool): + raise errors.BadConfigObject( + 'Configuration object is not an instance of CLITool') + + artifact_filters = cls._ParseStringOption(options, 'artifact_filters') + artifact_filters_file = cls._ParseStringOption( + options, 'artifact_filters_file') + filter_file = cls._ParseStringOption(options, 'file_filter') + + if artifact_filters and artifact_filters_file: + raise errors.BadConfigOption( + 'Please only specify artifact definition names in a file ' + 'or on the command line.') + + if (artifact_filters_file or artifact_filters) and filter_file: + raise errors.BadConfigOption( + 'Please do not specify both artifact definitions and legacy filters.') + + if artifact_filters_file and os.path.isfile(artifact_filters_file): + with open(artifact_filters_file) as file_object: + file_content = file_object.read() + artifact_filters = file_content.splitlines() + elif artifact_filters: + artifact_filters = [name.strip() for name in artifact_filters.split(',')] + + setattr(configuration_object, '_artifact_filters', artifact_filters) + + +manager.ArgumentHelperManager.RegisterHelper(ArtifactFiltersArgumentsHelper) diff --git a/plaso/cli/image_export_tool.py b/plaso/cli/image_export_tool.py index 9a4fc263f6..b7d5f68bc9 100644 --- a/plaso/cli/image_export_tool.py +++ b/plaso/cli/image_export_tool.py @@ -17,8 +17,8 @@ from plaso.cli import logger from plaso.cli import storage_media_tool from plaso.cli.helpers import manager as helpers_manager +from plaso.engine import engine from plaso.engine import extractors -from plaso.engine import filter_file from plaso.engine import knowledge_base from plaso.engine import path_helper from plaso.filters import file_entry as file_entry_filters @@ -73,7 +73,10 @@ def __init__(self, input_reader=None, output_writer=None): super(ImageExportTool, self).__init__( input_reader=input_reader, output_writer=output_writer) self._abort = False + self._artifact_definitions_path = None + self._artifact_filters = None self._artifacts_registry = None + self._custom_artifacts_path = None self._destination_path = None self._digests = {} self._filter_collection = file_entry_filters.FileEntryFilterCollection() @@ -287,7 +290,8 @@ def _ExtractFileEntry( # TODO: merge with collector and/or engine. def _ExtractWithFilter( self, source_path_specs, destination_path, output_writer, - filter_file_path, skip_duplicates=True): + artifact_filters, filter_file, artifact_definitions_path, + custom_artifacts_path, skip_duplicates=True): """Extracts files using a filter expression. This method runs the file extraction process on the image and @@ -297,8 +301,12 @@ def _ExtractWithFilter( source_path_specs (list[dfvfs.PathSpec]): path specifications to extract. destination_path (str): path where the extracted files should be stored. output_writer (CLIOutputWriter): output writer. - filter_file_path (str): path of the file that contains the filter - expressions. + artifact_definitions_path (str): path to artifact definitions file. + custom_artifacts_path (str): path to custom artifact definitions file. + artifact_filters (list[str]): names of artifact definitions that are + used for filtering file system and Windows Registry key paths. + filter_file (str): path of the file that contains the filter file path + filters. skip_duplicates (Optional[bool]): True if files with duplicate content should be skipped. """ @@ -314,14 +322,13 @@ def _ExtractWithFilter( output_writer.Write( 'Extracting file entries from: {0:s}\n'.format(display_name)) - environment_variables = self._knowledge_base.GetEnvironmentVariables() - filter_file_object = filter_file.FilterFile(filter_file_path) - find_specs = filter_file_object.BuildFindSpecs( - environment_variables=environment_variables) + filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs( + artifact_definitions_path, custom_artifacts_path, + self._knowledge_base, artifact_filters, filter_file) searcher = file_system_searcher.FileSystemSearcher( file_system, mount_point) - for path_spec in searcher.Find(find_specs=find_specs): + for path_spec in searcher.Find(find_specs=filter_find_specs): self._ExtractFileEntry( path_spec, destination_path, output_writer, skip_duplicates=skip_duplicates) @@ -399,7 +406,7 @@ def _ParseFilterOptions(self, options): Raises: BadConfigOption: if the options are invalid. """ - names = ['date_filters', 'filter_file'] + names = ['artifact_filters', 'date_filters', 'filter_file'] helpers_manager.ArgumentHelperManager.ParseOptions( options, self, names=names) @@ -416,7 +423,7 @@ def _ParseFilterOptions(self, options): except (IOError, ValueError) as exception: raise errors.BadConfigOption(exception) - if self._filter_file: + if self._artifact_filters or self._filter_file: self.has_filters = True else: self.has_filters = self._filter_collection.HasFilters() @@ -559,7 +566,7 @@ def AddFilterOptions(self, argument_group): Args: argument_group (argparse._ArgumentGroup): argparse argument group. """ - names = ['date_filters', 'filter_file'] + names = ['artifact_filters', 'date_filters', 'filter_file'] helpers_manager.ArgumentHelperManager.AddCommandLineArguments( argument_group, names=names) @@ -749,10 +756,12 @@ def ProcessSources(self): if not os.path.isdir(self._destination_path): os.makedirs(self._destination_path) - if self._filter_file: + if self._artifact_filters or self._filter_file: self._ExtractWithFilter( self._source_path_specs, self._destination_path, self._output_writer, - self._filter_file, skip_duplicates=self._skip_duplicates) + self._artifact_filters, self._filter_file, + self._artifact_definitions_path, self._custom_artifacts_path, + skip_duplicates=self._skip_duplicates) else: self._Extract( self._source_path_specs, self._destination_path, self._output_writer, diff --git a/plaso/cli/log2timeline_tool.py b/plaso/cli/log2timeline_tool.py index 13e9a7b732..e91130d588 100644 --- a/plaso/cli/log2timeline_tool.py +++ b/plaso/cli/log2timeline_tool.py @@ -23,7 +23,6 @@ from plaso.cli import views from plaso.cli.helpers import manager as helpers_manager from plaso.engine import engine -from plaso.engine import filter_file from plaso.engine import single_process as single_process_engine from plaso.lib import definitions from plaso.lib import errors @@ -167,7 +166,8 @@ def ParseArguments(self): 'extraction arguments') argument_helper_names = [ - 'extraction', 'filter_file', 'hashers', 'parsers', 'yara_rules'] + 'artifact_filters', 'extraction', 'filter_file', 'hashers', + 'parsers', 'yara_rules'] helpers_manager.ArgumentHelperManager.AddCommandLineArguments( extraction_group, names=argument_helper_names) @@ -317,8 +317,9 @@ def ParseOptions(self, options): self._ParseInformationalOptions(options) argument_helper_names = [ - 'artifact_definitions', 'extraction', 'filter_file', 'status_view', - 'storage_file', 'storage_format', 'text_prepend', 'yara_rules'] + 'artifact_definitions', 'artifact_filters', 'extraction', + 'filter_file', 'status_view', 'storage_file', 'storage_format', + 'text_prepend', 'yara_rules'] helpers_manager.ArgumentHelperManager.ParseOptions( options, self, names=argument_helper_names) @@ -370,7 +371,9 @@ def ExtractEventsFromSources(self): self._status_view.SetMode(self._status_view_mode) self._status_view.SetSourceInformation( - self._source_path, self._source_type, filter_file=self._filter_file) + self._source_path, self._source_type, + artifact_filters=self._artifact_filters, + filter_file=self._filter_file) status_update_callback = ( self._status_view.GetExtractionStatusUpdateCallback()) @@ -380,9 +383,10 @@ def ExtractEventsFromSources(self): self._output_writer.Write('Processing started.\n') session = engine.BaseEngine.CreateSession( + artifact_filter_names=self._artifact_filters, command_line_arguments=self._command_line_arguments, debug_mode=self._debug_mode, - filter_file=self._filter_file, + filter_file_path=self._filter_file, preferred_encoding=self.preferred_encoding, preferred_time_zone=self._preferred_time_zone, preferred_year=self._preferred_year) @@ -415,13 +419,15 @@ def ExtractEventsFromSources(self): self._SetExtractionParsersAndPlugins(configuration, session) self._SetExtractionPreferredTimeZone(extraction_engine.knowledge_base) - filter_find_specs = None - if configuration.filter_file: - environment_variables = ( - extraction_engine.knowledge_base.GetEnvironmentVariables()) - filter_file_object = filter_file.FilterFile(configuration.filter_file) - filter_find_specs = filter_file_object.BuildFindSpecs( - environment_variables=environment_variables) + artifact_filters = getattr(configuration, '_artifact_filters', None) + artifact_definitions_path = getattr( + configuration, '_artifact_definitions_path', None) + custom_artifacts_path = getattr( + configuration, '_custom_artifacts_path', None) + filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs( + artifact_definitions_path, custom_artifacts_path, + extraction_engine.knowledge_base, artifact_filters, + configuration.filter_file) processing_status = None if single_process_mode: diff --git a/plaso/cli/pinfo_tool.py b/plaso/cli/pinfo_tool.py index 7e83032f56..c5142d5e00 100644 --- a/plaso/cli/pinfo_tool.py +++ b/plaso/cli/pinfo_tool.py @@ -353,6 +353,7 @@ def _PrintSessionsDetails(self, storage): command_line_arguments = session.command_line_arguments or 'N/A' parser_filter_expression = session.parser_filter_expression or 'N/A' preferred_encoding = session.preferred_encoding or 'N/A' + artifact_filters = session.artifact_filters or 'N/A' filter_file = session.filter_file or 'N/A' title = 'Session: {0!s}'.format(session_identifier) @@ -368,6 +369,7 @@ def _PrintSessionsDetails(self, storage): table_view.AddRow(['Enabled parser and plugins', enabled_parser_names]) table_view.AddRow(['Preferred encoding', preferred_encoding]) table_view.AddRow(['Debug mode', session.debug_mode]) + table_view.AddRow(['Artifact filters', artifact_filters]) table_view.AddRow(['Filter file', filter_file]) table_view.Write(self._output_writer) diff --git a/plaso/cli/psteal_tool.py b/plaso/cli/psteal_tool.py index 1f57cc5990..a98021e9c6 100644 --- a/plaso/cli/psteal_tool.py +++ b/plaso/cli/psteal_tool.py @@ -22,7 +22,6 @@ from plaso.cli import views from plaso.cli.helpers import manager as helpers_manager from plaso.engine import engine -from plaso.engine import filter_file from plaso.engine import knowledge_base from plaso.engine import single_process as single_process_engine from plaso.lib import errors @@ -273,7 +272,9 @@ def ExtractEventsFromSources(self): self._status_view.SetMode(self._status_view_mode) self._status_view.SetSourceInformation( - self._source_path, source_type, filter_file=self._filter_file) + self._source_path, source_type, + artifact_filters=self._artifact_filters, + filter_file=self._filter_file) status_update_callback = ( self._status_view.GetExtractionStatusUpdateCallback()) @@ -283,8 +284,9 @@ def ExtractEventsFromSources(self): self._output_writer.Write('Processing started.\n') session = engine.BaseEngine.CreateSession( + artifact_filter_names=self._artifact_filters, command_line_arguments=self._command_line_arguments, - filter_file=self._filter_file, + filter_file_path=self._filter_file, preferred_encoding=self.preferred_encoding, preferred_time_zone=self._preferred_time_zone, preferred_year=self._preferred_year) @@ -317,13 +319,15 @@ def ExtractEventsFromSources(self): self._SetExtractionParsersAndPlugins(configuration, session) self._SetExtractionPreferredTimeZone(extraction_engine.knowledge_base) - filter_find_specs = None - if configuration.filter_file: - environment_variables = ( - extraction_engine.knowledge_base.GetEnvironmentVariables()) - filter_file_object = filter_file.FilterFile(configuration.filter_file) - filter_find_specs = filter_file_object.BuildFindSpecs( - environment_variables=environment_variables) + artifact_filters = getattr(configuration, '_artifact_filters', None) + artifact_definitions_path = getattr( + configuration, '_artifact_definitions_path', None) + custom_artifacts_path = getattr( + configuration, '_custom_artifacts_path', None) + filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs( + artifact_definitions_path, custom_artifacts_path, + extraction_engine.knowledge_base, artifact_filters, + configuration.filter_file) processing_status = None if single_process_mode: @@ -450,7 +454,8 @@ def ParseOptions(self, options): self._ParseTimezoneOption(options) argument_helper_names = [ - 'artifact_definitions', 'hashers', 'language', 'parsers'] + 'artifact_definitions', 'custom_artifact_definitions', + 'hashers', 'language', 'parsers'] helpers_manager.ArgumentHelperManager.ParseOptions( options, self, names=argument_helper_names) diff --git a/plaso/cli/status_view.py b/plaso/cli/status_view.py index 0bba8fe97a..deb1305f2d 100644 --- a/plaso/cli/status_view.py +++ b/plaso/cli/status_view.py @@ -45,6 +45,7 @@ def __init__(self, output_writer, tool_name): tool_name (str): namd of the tool. """ super(StatusView, self).__init__() + self._artifact_filters = None self._filter_file = None self._mode = self.MODE_WINDOW self._output_writer = output_writer @@ -318,6 +319,9 @@ def PrintExtractionStatusHeader(self, processing_status): self._output_writer.Write( 'Source type\t: {0:s}\n'.format(self._source_type)) + if self._artifact_filters: + self._output_writer.Write('Artifact filters\t: {0:s}\n'.format( + self._artifact_filters)) if self._filter_file: self._output_writer.Write('Filter file\t: {0:s}\n'.format( self._filter_file)) @@ -400,14 +404,18 @@ def SetMode(self, mode): """ self._mode = mode - def SetSourceInformation(self, source_path, source_type, filter_file=None): + def SetSourceInformation( + self, source_path, source_type, artifact_filters=None, filter_file=None): """Sets the source information. Args: source_path (str): path of the source. source_type (str): source type. + artifact_filters (Optional[str]): names of artifact defintions to use as + filters. filter_file (Optional[str]): filter file. """ + self._artifact_filters = artifact_filters self._filter_file = filter_file self._source_path = source_path self._source_type = self._SOURCE_TYPES.get(source_type, 'UNKNOWN') diff --git a/plaso/cli/storage_media_tool.py b/plaso/cli/storage_media_tool.py index 0b8da0a90c..29ddbe72c5 100644 --- a/plaso/cli/storage_media_tool.py +++ b/plaso/cli/storage_media_tool.py @@ -63,6 +63,7 @@ def __init__(self, input_reader=None, output_writer=None): """ super(StorageMediaTool, self).__init__( input_reader=input_reader, output_writer=output_writer) + self._artifact_filters = None self._credentials = [] self._credential_configurations = [] self._filter_file = None diff --git a/plaso/containers/sessions.py b/plaso/containers/sessions.py index eac9df76c7..e64248dfbb 100644 --- a/plaso/containers/sessions.py +++ b/plaso/containers/sessions.py @@ -19,6 +19,8 @@ class Session(interface.AttributeContainer): aborted (bool): True if the session was aborted. analysis_reports_counter (collections.Counter): number of analysis reports per analysis plugin. + artifact_filters (list[str]): Names of artifact definitions that are + used for filtering file system and Windows Registry key paths. command_line_arguments (str): command line arguments. completion_time (int): time that the session was completed. Contains the number of micro seconds since January 1, 1970, 00:00:00 UTC. @@ -47,6 +49,7 @@ def __init__(self): super(Session, self).__init__() self.aborted = False self.analysis_reports_counter = collections.Counter() + self.artifact_filters = None self.command_line_arguments = None self.completion_time = None self.debug_mode = False @@ -97,6 +100,7 @@ def CopyAttributesFromSessionStart(self, session_start): Args: session_start (SessionStart): session start attribute container. """ + self.artifact_filters = session_start.artifact_filters self.command_line_arguments = session_start.command_line_arguments self.debug_mode = session_start.debug_mode self.enabled_parser_names = session_start.enabled_parser_names @@ -133,6 +137,7 @@ def CreateSessionStart(self): SessionStart: session start attribute container. """ session_start = SessionStart() + session_start.artifact_filters = self.artifact_filters session_start.command_line_arguments = self.command_line_arguments session_start.debug_mode = self.debug_mode session_start.enabled_parser_names = self.enabled_parser_names @@ -184,6 +189,8 @@ class SessionStart(interface.AttributeContainer): """Session start attribute container. Attributes: + artifact_filters (list[str]): names of artifact definitions that are + used for filtering file system and Windows Registry key paths. command_line_arguments (str): command line arguments. debug_mode (bool): True if debug mode was enabled. enabled_parser_names (list[str]): parser and parser plugin names that @@ -211,6 +218,7 @@ def __init__(self, identifier=None): session completion information. """ super(SessionStart, self).__init__() + self.artifact_filters = None self.command_line_arguments = None self.debug_mode = False self.enabled_parser_names = None diff --git a/plaso/engine/artifact_filters.py b/plaso/engine/artifact_filters.py index 7a3d91ff78..a038e77067 100644 --- a/plaso/engine/artifact_filters.py +++ b/plaso/engine/artifact_filters.py @@ -20,27 +20,33 @@ class ArtifactDefinitionsFilterHelper(object): https://github.com/ForensicArtifacts/artifacts/blob/master/docs/Artifacts%20definition%20format%20and%20style%20guide.asciidoc """ - _KNOWLEDGE_BASE_VALUE = 'ARTIFACT_FILTERS' + KNOWLEDGE_BASE_VALUE = 'ARTIFACT_FILTERS' - _COMPATIBLE_REGISTRY_KEY_PATH_PREFIXES = ['HKEY_LOCAL_MACHINE'] + _COMPATIBLE_REGISTRY_KEY_PATH_PREFIXES = frozenset([ + 'HKEY_LOCAL_MACHINE\\SYSTEM', + 'HKEY_LOCAL_MACHINE\\SOFTWARE', + 'HKEY_LOCAL_MACHINE\\SAM', + 'HKEY_LOCAL_MACHINE\\SECURITY']) - def __init__(self, artifacts_registry, artifact_definitions, knowledge_base): + def __init__(self, artifacts_registry, artifact_filters, knowledge_base): """Initializes an artifact definitions filter helper. Args: artifacts_registry (artifacts.ArtifactDefinitionsRegistry]): artifact definitions registry. - artifact_definitions (list[str]): artifact definition names to filter. + artifact_filters (list[str]): names of artifact definitions that are + used for filtering file system and Windows Registry key paths. path (str): path to a file that contains one or more artifact definitions. knowledge_base (KnowledgeBase): contains information from the source data needed for filtering. """ super(ArtifactDefinitionsFilterHelper, self).__init__() - self._artifacts = artifact_definitions + self._artifacts = artifact_filters self._artifacts_registry = artifacts_registry self._knowledge_base = knowledge_base - def _CheckKeyCompatibility(self, key_path): + @staticmethod + def CheckKeyCompatibility(key_path): """Checks if a Windows Registry key path is supported by dfWinReg. Args: @@ -49,7 +55,9 @@ def _CheckKeyCompatibility(self, key_path): Returns: bool: True if key is compatible or False if not. """ - for key_path_prefix in self._COMPATIBLE_REGISTRY_KEY_PATH_PREFIXES: + for key_path_prefix in ( + ArtifactDefinitionsFilterHelper._COMPATIBLE_REGISTRY_KEY_PATH_PREFIXES): + key_path = key_path.upper() if key_path.startswith(key_path_prefix): return True @@ -91,7 +99,7 @@ def BuildFindSpecs(self, environment_variables=None): # TODO: move source.keys iteration into # BuildFindSpecsFromRegistryArtifact. for key_path in set(source.keys): - if self._CheckKeyCompatibility(key_path): + if self.CheckKeyCompatibility(key_path): find_specs = self.BuildFindSpecsFromRegistryArtifact(key_path) find_specs_per_source_type[ artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY].extend( @@ -108,8 +116,8 @@ def BuildFindSpecs(self, environment_variables=None): # TODO: move source.key_value_pairs iteration into # BuildFindSpecsFromRegistryArtifact. for key_path in set([ - key_path for key_path, _ in source.key_value_pairs]): - if self._CheckKeyCompatibility(key_path): + key_value['key'] for key_value in source.key_value_pairs]): + if self.CheckKeyCompatibility(key_path): find_specs = self.BuildFindSpecsFromRegistryArtifact(key_path) find_specs_per_source_type[ artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY].extend( @@ -121,7 +129,7 @@ def BuildFindSpecs(self, environment_variables=None): source.type_indicator)) self._knowledge_base.SetValue( - self._KNOWLEDGE_BASE_VALUE, find_specs_per_source_type) + self.KNOWLEDGE_BASE_VALUE, find_specs_per_source_type) def BuildFindSpecsFromFileArtifact( self, source_path, path_separator, environment_variables, user_accounts): diff --git a/plaso/engine/configurations.py b/plaso/engine/configurations.py index 4c36c90cf7..a837651600 100644 --- a/plaso/engine/configurations.py +++ b/plaso/engine/configurations.py @@ -190,6 +190,9 @@ class ProcessingConfiguration(interface.AttributeContainer): """Configuration settings for processing. Attributes: + artifact_filters (Optional list[str]): names of artifact + definitions that are used for filtering file system and Windows + Registry key paths. credentials (list[CredentialConfiguration]): credential configurations. data_location (str): path to the data files. debug_output (bool): True if debug output should be enabled. @@ -211,6 +214,7 @@ class ProcessingConfiguration(interface.AttributeContainer): def __init__(self): """Initializes a process configuration object.""" super(ProcessingConfiguration, self).__init__() + self.artifact_filters = None self.credentials = [] self.data_location = None self.debug_output = False diff --git a/plaso/engine/engine.py b/plaso/engine/engine.py index a1813f25a2..37f2a83117 100644 --- a/plaso/engine/engine.py +++ b/plaso/engine/engine.py @@ -3,17 +3,27 @@ from __future__ import unicode_literals +import os + +from artifacts import definitions as artifact_types +from artifacts import errors as artifacts_errors +from artifacts import reader as artifacts_reader +from artifacts import registry as artifacts_registry + from dfvfs.helpers import file_system_searcher from dfvfs.lib import errors as dfvfs_errors from dfvfs.path import factory as path_spec_factory from dfvfs.resolver import resolver as path_spec_resolver from plaso.containers import sessions +from plaso.engine import artifact_filters +from plaso.engine import filter_file from plaso.engine import knowledge_base from plaso.engine import logger from plaso.engine import processing_status from plaso.engine import profilers from plaso.lib import definitions +from plaso.lib import errors from plaso.preprocessors import manager as preprocess_manager @@ -162,15 +172,18 @@ def _StopProfiling(self): @classmethod def CreateSession( - cls, command_line_arguments=None, debug_mode=False, - filter_file=None, preferred_encoding='utf-8', + cls, artifact_filter_names=None, command_line_arguments=None, + debug_mode=False, filter_file_path=None, preferred_encoding='utf-8', preferred_time_zone=None, preferred_year=None): """Creates a session attribute container. Args: + artifact_filter_names (Optional list[str]): names of artifact definitions + that are used for filtering file system and Windows Registry + key paths. command_line_arguments (Optional[str]): the command line arguments. debug_mode (bool): True if debug mode was enabled. - filter_file (Optional[str]): path to a file with find specifications. + filter_file_path (Optional[str]): path to a file with find specifications. preferred_encoding (Optional[str]): preferred encoding. preferred_time_zone (Optional[str]): preferred time zone. preferred_year (Optional[int]): preferred year. @@ -180,9 +193,10 @@ def CreateSession( """ session = sessions.Session() + session.artifact_filters = artifact_filter_names session.command_line_arguments = command_line_arguments session.debug_mode = debug_mode - session.filter_file = filter_file + session.filter_file = filter_file_path session.preferred_encoding = preferred_encoding session.preferred_time_zone = preferred_time_zone session.preferred_year = preferred_year @@ -225,12 +239,13 @@ def GetSourceFileSystem(self, source_path_spec, resolver_context=None): return file_system, mount_point def PreprocessSources( - self, artifacts_registry, source_path_specs, resolver_context=None): + self, artifacts_registry_object, source_path_specs, + resolver_context=None): """Preprocesses the sources. Args: - artifacts_registry (artifacts.ArtifactDefinitionsRegistry]): artifact - definitions registry. + artifacts_registry_object (artifacts.ArtifactDefinitionsRegistry]): + artifact definitions registry. source_path_specs (list[dfvfs.PathSpec]): path specifications of the sources to process. resolver_context (Optional[dfvfs.Context]): resolver context. @@ -251,7 +266,8 @@ def PreprocessSources( operating_system = self._DetermineOperatingSystem(searcher) if operating_system != definitions.OPERATING_SYSTEM_UNKNOWN: preprocess_manager.PreprocessPluginsManager.RunPlugins( - artifacts_registry, file_system, mount_point, self.knowledge_base) + artifacts_registry_object, file_system, mount_point, + self.knowledge_base) detected_operating_systems.append(operating_system) @@ -272,3 +288,93 @@ def SupportsGuppyMemoryProfiling(cls): bool: True if memory profiling with guppy is supported. """ return profilers.GuppyMemoryProfiler.IsSupported() + + @classmethod + def BuildFilterFindSpecs( + cls, artifact_defintions_path, custom_artifacts_path, + knowledge_base_object, artifact_filter_names=None, filter_file_path=None): + """Builds find specifications from artifacts or filter file if available. + + Args: + knowledge_base_object (KnowledgeBase): knowledge base. + artifact_filter_names (Optional list[str]): names of artifact + definitions that are used for filtering file system and Windows + Registry key paths. + filter_file_path (Optional [str]): Path of filter file. + + Returns: + list[dfvfs.FindSpec]: find specifications for the file source type. + + Raises: + RuntimeError: if no valid FindSpecs are built. + """ + environment_variables = knowledge_base_object.GetEnvironmentVariables() + find_specs = None + if artifact_filter_names: + artifacts_registry_object = cls.BuildArtifactsRegistry( + artifact_defintions_path, custom_artifacts_path) + artifact_filters_object = ( + artifact_filters.ArtifactDefinitionsFilterHelper( + artifacts_registry_object, artifact_filter_names, + knowledge_base_object)) + artifact_filters_object.BuildFindSpecs(environment_variables) + find_specs = knowledge_base_object.GetValue( + artifact_filters_object.KNOWLEDGE_BASE_VALUE)[ + artifact_types.TYPE_INDICATOR_FILE] + elif filter_file_path: + filter_file_object = filter_file.FilterFile(filter_file_path) + find_specs = filter_file_object.BuildFindSpecs( + environment_variables=environment_variables) + + if (artifact_filter_names or filter_file_path) and not find_specs: + raise RuntimeError( + 'Error processing filters, no valid specifications built.') + + return find_specs + + @classmethod + def BuildArtifactsRegistry( + cls, artifacts_definitions_path, custom_artifacts_path): + """Build Find Specs from artifacts or filter file if available. + + Args: + artifact_definitions_path (str): path to artifact definitions file. + custom_artifacts_path (str): path to custom artifact definitions file. + + Returns: + artifacts.ArtifactDefinitionsRegistry: artifact definitions registry. + + Raises: + RuntimeError: if no valid FindSpecs are built. + """ + if artifacts_definitions_path and not os.path.isdir( + artifacts_definitions_path): + raise errors.BadConfigOption( + 'No such artifacts filter file: {0:s}.'.format( + artifacts_definitions_path)) + + if custom_artifacts_path and not os.path.isfile(custom_artifacts_path): + raise errors.BadConfigOption( + 'No such artifacts filter file: {0:s}.'.format(custom_artifacts_path)) + + registry = artifacts_registry.ArtifactDefinitionsRegistry() + reader = artifacts_reader.YamlArtifactsReader() + + try: + registry.ReadFromDirectory(reader, artifacts_definitions_path) + + except (KeyError, artifacts_errors.FormatError) as exception: + raise errors.BadConfigOption(( + 'Unable to read artifact definitions from: {0:s} with error: ' + '{1!s}').format(artifacts_definitions_path, exception)) + + if custom_artifacts_path: + try: + registry.ReadFromFile(reader, custom_artifacts_path) + + except (KeyError, artifacts_errors.FormatError) as exception: + raise errors.BadConfigOption(( + 'Unable to read artifact definitions from: {0:s} with error: ' + '{1!s}').format(custom_artifacts_path, exception)) + + return registry diff --git a/plaso/parsers/winreg.py b/plaso/parsers/winreg.py index 5cb4a6e252..b4fdcca632 100644 --- a/plaso/parsers/winreg.py +++ b/plaso/parsers/winreg.py @@ -3,11 +3,14 @@ from __future__ import unicode_literals +from artifacts import definitions as artifact_types from dfwinreg import errors as dfwinreg_errors from dfwinreg import interface as dfwinreg_interface from dfwinreg import regf as dfwinreg_regf from dfwinreg import registry as dfwinreg_registry +from dfwinreg import registry_searcher as dfwinreg_registry_searcher +from plaso.engine import artifact_filters from plaso.lib import specification from plaso.filters import path_filter from plaso.parsers import interface @@ -158,6 +161,30 @@ def _NormalizeKeyPath(self, key_path): return ''.join([ self._NORMALIZED_CONTROL_SET_PREFIX, normalized_key_path[39:]]) + def _ParseKey(self, parser_mediator, registry_key): + """Parses the Registry key with a specific plugin. + + Args: + parser_mediator (ParserMediator): parser mediator. + registry_key (dfwinreg.WinRegistryKey): Windwos Registry key. + """ + matching_plugin = None + + normalized_key_path = self._NormalizeKeyPath(registry_key.path) + if self._path_filter.CheckPath(normalized_key_path): + matching_plugin = self._plugin_per_key_path[normalized_key_path] + else: + for plugin in self._plugins_without_key_paths: + if self._CanProcessKeyWithPlugin(registry_key, plugin): + matching_plugin = plugin + break + + if not matching_plugin: + matching_plugin = self._default_plugin + + if matching_plugin: + self._ParseKeyWithPlugin(parser_mediator, registry_key, matching_plugin) + def _ParseRecurseKeys(self, parser_mediator, root_key): """Parses the Registry keys recursively. @@ -169,23 +196,25 @@ def _ParseRecurseKeys(self, parser_mediator, root_key): if parser_mediator.abort: break - matching_plugin = None + self._ParseKey(parser_mediator, registry_key) - normalized_key_path = self._NormalizeKeyPath(registry_key.path) - if self._path_filter.CheckPath(normalized_key_path): - matching_plugin = self._plugin_per_key_path[normalized_key_path] - else: - for plugin in self._plugins_without_key_paths: - if self._CanProcessKeyWithPlugin(registry_key, plugin): - matching_plugin = plugin - break + def _ParseKeysFromFindSpecs(self, parser_mediator, win_registry, find_specs): + """Parses the Registry keys from FindSpecs. - if not matching_plugin: - matching_plugin = self._default_plugin + Args: + parser_mediator (ParserMediator): parser mediator. + win_registry (dfwinreg.WinRegistryKey): root Windows Registry key. + find_specs (dfwinreg.FindSpecs): Keys to search for. + """ + searcher = dfwinreg_registry_searcher.WinRegistrySearcher(win_registry) + for registry_key_path in iter(searcher.Find(find_specs=find_specs)): + if parser_mediator.abort: + break + + registry_key = searcher.GetKeyByPath(registry_key_path) + self._ParseKey(parser_mediator, registry_key) - if matching_plugin: - self._ParseKeyWithPlugin(parser_mediator, registry_key, matching_plugin) def ParseFileObject(self, parser_mediator, file_object, **kwargs): """Parses a Windows Registry file-like object. @@ -205,16 +234,40 @@ def ParseFileObject(self, parser_mediator, file_object, **kwargs): return win_registry = dfwinreg_registry.WinRegistry() + key_path_prefix = win_registry.GetRegistryFileMapping(registry_file) registry_file.SetKeyPathPrefix(key_path_prefix) root_key = registry_file.GetRootKey() if not root_key: return - try: - self._ParseRecurseKeys(parser_mediator, root_key) - except IOError as exception: - parser_mediator.ProduceExtractionError('{0:s}'.format(exception)) + find_specs = parser_mediator.knowledge_base.GetValue( + artifact_filters.ArtifactDefinitionsFilterHelper.KNOWLEDGE_BASE_VALUE) + + registry_find_specs = None + if find_specs: + registry_find_specs = find_specs.get( + artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY) + + key_path_compatible = (artifact_filters.ArtifactDefinitionsFilterHelper. + CheckKeyCompatibility(key_path_prefix)) + + if registry_find_specs and key_path_compatible: + try: + win_registry.MapFile(key_path_prefix, registry_file) + self._ParseKeysFromFindSpecs( + parser_mediator, win_registry, registry_find_specs) + except IOError as exception: + parser_mediator.ProduceExtractionError('{0:s}'.format(exception)) + else: + if registry_find_specs and not key_path_compatible: + logger.warning('Artifacts Registry Filters are not supported for ' + 'the registry prefix {0:s}. ' + 'Parsing entire file.'.format(key_path_prefix)) + try: + self._ParseRecurseKeys(parser_mediator, root_key) + except IOError as exception: + parser_mediator.ProduceExtractionError('{0:s}'.format(exception)) manager.ParsersManager.RegisterParser(WinRegistryParser) diff --git a/test_data/artifacts/artifact_names b/test_data/artifacts/artifact_names new file mode 100644 index 0000000000..d466270af8 --- /dev/null +++ b/test_data/artifacts/artifact_names @@ -0,0 +1,3 @@ +TestFiles +TestFiles2 +TestFiles3 diff --git a/test_data/artifacts/artifacts_filters.yaml b/test_data/artifacts/artifacts_filters.yaml index 7de1896d7f..e1dd18ecd9 100644 --- a/test_data/artifacts/artifacts_filters.yaml +++ b/test_data/artifacts/artifacts_filters.yaml @@ -40,10 +40,9 @@ sources: - type: REGISTRY_KEY attributes: keys: - - 'HKEY_LOCAL_MACHINE\System\ControlSet001\services\**\' - - 'HKEY_LOCAL_MACHINE\System\ControlSet002\services\**\' + - 'HKEY_LOCAL_MACHINE\System\ControlSet001\services\**' + - 'HKEY_LOCAL_MACHINE\System\ControlSet002\services\**' - 'HKEY_LOCAL_MACHINE\System\CurrentControlSet\Enum\USBSTOR' - - 'HKEY_LOCAL_MACHINE\System\CurrentControlSet\Enum\USBSTOR\**' supported_os: [Windows] --- name: TestRegistryValue diff --git a/tests/cli/helpers/artifact_definitions.py b/tests/cli/helpers/artifact_definitions.py index 411708c0bf..9e334e10a2 100644 --- a/tests/cli/helpers/artifact_definitions.py +++ b/tests/cli/helpers/artifact_definitions.py @@ -22,6 +22,7 @@ class ArtifactDefinitionsArgumentsHelperTest(cli_test_lib.CLIToolTestCase): _EXPECTED_OUTPUT = """\ usage: cli_helper.py [--artifact_definitions PATH] + [--custom_artifact_definitions PATH] Test argument parser. @@ -31,6 +32,11 @@ class ArtifactDefinitionsArgumentsHelperTest(cli_test_lib.CLIToolTestCase): which are .yaml files. Artifact definitions can be used to describe and quickly collect data of interest, such as specific files or Windows Registry keys. + --custom_artifact_definitions PATH, --custom-artifact-definitions PATH + Path to a file containing custom artifact definitions, + which are .yaml files. Artifact definitions can be + used to describe and quickly collect data of interest, + such as specific files or Windows Registry keys. """ def testAddArguments(self): @@ -56,7 +62,7 @@ def testParseOptions(self): artifact_definitions.ArtifactDefinitionsArgumentsHelper.ParseOptions( options, test_tool) - self.assertIsNotNone(test_tool._artifacts_registry) + self.assertIsNotNone(test_tool._artifact_definitions_path) with self.assertRaises(errors.BadConfigObject): artifact_definitions.ArtifactDefinitionsArgumentsHelper.ParseOptions( diff --git a/tests/cli/helpers/artifact_filters.py b/tests/cli/helpers/artifact_filters.py new file mode 100644 index 0000000000..3c97cd30cc --- /dev/null +++ b/tests/cli/helpers/artifact_filters.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Tests for the filter file CLI arguments helper.""" + +from __future__ import unicode_literals + +import argparse +import unittest + +from plaso.cli import tools +from plaso.cli.helpers import artifact_filters +from plaso.lib import errors + +from tests.cli import test_lib as cli_test_lib + + +class ArtifactFiltersArgumentsHelperTest(cli_test_lib.CLIToolTestCase): + """Tests for the filter file CLI arguments helper.""" + + # pylint: disable=no-member,protected-access + + _EXPECTED_OUTPUT = """\ +usage: cli_helper.py [--artifact_filters ARTIFACT_FILTERS] + [--artifact_filters_file ARTIFACT_FILTERS] + +Test argument parser. + +optional arguments: + --artifact_filters ARTIFACT_FILTERS, --artifact-filters ARTIFACT_FILTERS + Names of forensic artifact definitions, provided on + the command command line (comma separated). Forensic + artifacts are stored in .yaml files that are directly + pulled from the artifact definitions project. You can + also specify a custom artifacts yaml file (see + --custom_artifact_definitions). Artifact definitions + can be used to describe and quickly collect data of + interest, such as specific files or Windows Registry + keys. + --artifact_filters_file ARTIFACT_FILTERS, --artifact-filters_file ARTIFACT_FILTERS + Names of forensic artifact definitions, provided in a + file with one artifact name per line. Forensic + artifacts are stored in .yaml files that are directly + pulled from the artifact definitions project. You can + also specify a custom artifacts yaml file (see + --custom_artifact_definitions). Artifact definitions + can be used to describe and quickly collect data of + interest, such as specific files or Windows Registry + keys. +""" + + def testAddArguments(self): + """Tests the AddArguments function.""" + argument_parser = argparse.ArgumentParser( + prog='cli_helper.py', description='Test argument parser.', + add_help=False, + formatter_class=cli_test_lib.SortedArgumentsHelpFormatter) + + artifact_filters.ArtifactFiltersArgumentsHelper.AddArguments( + argument_parser) + + output = self._RunArgparseFormatHelp(argument_parser) + self.assertEqual(output, self._EXPECTED_OUTPUT) + + def testParseOptions(self): + """Tests the ParseOptions function.""" + options = cli_test_lib.TestOptions() + options.artifact_filters = 'TestFiles, TestFiles2' + expected_output = ['TestFiles', 'TestFiles2'] + + test_tool = tools.CLITool() + artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( + options, test_tool) + + self.assertItemsEqual(test_tool._artifact_filters, expected_output) + + options.artifact_filters_file = self._GetTestFilePath( + ['artifacts', 'artifact_names']) + + with self.assertRaises(errors.BadConfigOption): + artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( + options, test_tool) + + expected_output = ['TestFiles', 'TestFiles2', 'TestFiles3'] + + options.artifact_filters = None + artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( + options, test_tool) + + self.assertItemsEqual(test_tool._artifact_filters, expected_output) + + options.file_filter = self._GetTestFilePath(['testdir', 'filter2.txt']) + with self.assertRaises(errors.BadConfigOption): + artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( + options, test_tool) + + with self.assertRaises(errors.BadConfigObject): + artifact_filters.ArtifactFiltersArgumentsHelper.ParseOptions( + options, None) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/cli/image_export_tool.py b/tests/cli/image_export_tool.py index 85417730b3..9e2a7d5a8e 100644 --- a/tests/cli/image_export_tool.py +++ b/tests/cli/image_export_tool.py @@ -398,6 +398,35 @@ def testProcessSourcesExtractWithFilter(self): self.assertEqual(sorted(extracted_files), expected_extracted_files) + @shared_test_lib.skipUnlessHasTestFile(['artifacts']) + @shared_test_lib.skipUnlessHasTestFile(['image.qcow2']) + def testProcessSourcesExtractWithArtifactsFilter(self): + """Tests the ProcessSources function with a filter file.""" + output_writer = test_lib.TestOutputWriter(encoding='utf-8') + test_tool = image_export_tool.ImageExportTool(output_writer=output_writer) + + options = test_lib.TestOptions() + options.artifact_definitions_path = self._GetTestFilePath(['artifacts']) + options.image = self._GetTestFilePath(['image.qcow2']) + options.quiet = True + options.artifact_filters = 'TestFilesImageExport' + + with shared_test_lib.TempDirectory() as temp_directory: + options.path = temp_directory + + test_tool.ParseOptions(options) + + test_tool.ProcessSources() + + expected_extracted_files = sorted([ + os.path.join(temp_directory, 'a_directory'), + os.path.join(temp_directory, 'a_directory', 'another_file'), + os.path.join(temp_directory, 'a_directory', 'a_file')]) + + extracted_files = self._RecursiveList(temp_directory) + + self.assertEqual(sorted(extracted_files), expected_extracted_files) + @shared_test_lib.skipUnlessHasTestFile(['syslog_image.dd']) def testProcessSourcesExtractWithSignaturesFilter(self): """Tests the ProcessSources function with a signatures filter.""" diff --git a/tests/cli/pinfo_tool.py b/tests/cli/pinfo_tool.py index 03d27376a5..0712ee1585 100644 --- a/tests/cli/pinfo_tool.py +++ b/tests/cli/pinfo_tool.py @@ -185,6 +185,7 @@ def testPrintStorageInformationAsText(self): table_view.AddRow(['Enabled parser and plugins', enabled_parser_names]) table_view.AddRow(['Preferred encoding', 'UTF-8']) table_view.AddRow(['Debug mode', 'False']) + table_view.AddRow(['Artifact filters', 'N/A']) table_view.AddRow(['Filter file', 'N/A']) table_view.Write(output_writer) diff --git a/tests/cli/psteal_tool.py b/tests/cli/psteal_tool.py index e2fb0c5ea2..6efcf202bc 100644 --- a/tests/cli/psteal_tool.py +++ b/tests/cli/psteal_tool.py @@ -119,6 +119,7 @@ def testParseOptions(self): test_tool = psteal_tool.PstealTool(output_writer=output_writer) options = test_lib.TestOptions() + options.artifact_definitions_path = self._GetTestFilePath(['artifacts']) options.source = 'source' # Test when the output file is missing. expected_error = 'Output format: dynamic requires an output file' diff --git a/tests/engine/artifact_filters.py b/tests/engine/artifact_filters.py index 41a43e0e88..aa6365f0cb 100644 --- a/tests/engine/artifact_filters.py +++ b/tests/engine/artifact_filters.py @@ -81,7 +81,7 @@ def testBuildFindSpecsWithFileSystem(self): test_filter_file.BuildFindSpecs( environment_variables=[environment_variable]) find_specs_per_source_type = knowledge_base.GetValue( - test_filter_file._KNOWLEDGE_BASE_VALUE) + test_filter_file.KNOWLEDGE_BASE_VALUE) find_specs = find_specs_per_source_type.get( artifact_types.TYPE_INDICATOR_FILE, []) @@ -122,7 +122,7 @@ def testBuildFindSpecsWithRegistry(self): test_filter_file.BuildFindSpecs(environment_variables=None) find_specs_per_source_type = knowledge_base.GetValue( - test_filter_file._KNOWLEDGE_BASE_VALUE) + test_filter_file.KNOWLEDGE_BASE_VALUE) find_specs = find_specs_per_source_type.get( artifact_types.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY, []) @@ -150,19 +150,19 @@ def testBuildFindSpecsWithRegistry(self): self.assertEqual(len(key_paths), 3) def testCheckKeyCompatibility(self): - """Tests the _CheckKeyCompatibility function""" + """Tests the CheckKeyCompatibility function""" knowledge_base = knowledge_base_engine.KnowledgeBase() test_filter_file = self._CreateTestArtifactDefinitionsFilterHelper( [], knowledge_base) # Compatible Key. key_path = 'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control' - compatible_key = test_filter_file._CheckKeyCompatibility(key_path) + compatible_key = test_filter_file.CheckKeyCompatibility(key_path) self.assertTrue(compatible_key) # NOT a Compatible Key. key_path = 'HKEY_USERS\\S-1-5-18' - compatible_key = test_filter_file._CheckKeyCompatibility(key_path) + compatible_key = test_filter_file.CheckKeyCompatibility(key_path) self.assertFalse(compatible_key) def testBuildFindSpecsFromFileArtifact(self): diff --git a/tests/parsers/test_lib.py b/tests/parsers/test_lib.py index 6c9427b8c5..6b6646c98d 100644 --- a/tests/parsers/test_lib.py +++ b/tests/parsers/test_lib.py @@ -134,7 +134,10 @@ def _ParseFileByPathSpec( try: parser.Parse(parser_mediator, file_object) finally: - file_object.close() + # Check if file_object is open as dfwinreg will have closed file_object. + # TODO: Fix once Issue #306 addressed in DFVFS + if file_object._is_open: # pylint: disable=protected-access + file_object.close() else: self.fail('Got unsupported parser type: {0:s}'.format(type(parser))) diff --git a/tests/parsers/winreg.py b/tests/parsers/winreg.py index 82856573ea..b8e7cc98f9 100644 --- a/tests/parsers/winreg.py +++ b/tests/parsers/winreg.py @@ -6,6 +6,11 @@ import unittest +from artifacts import reader as artifacts_reader +from artifacts import registry as artifacts_registry + +from plaso.engine import artifact_filters +from plaso.engine import knowledge_base as knowledge_base_engine from plaso.parsers import winreg # Register all plugins. from plaso.parsers import winreg_plugins # pylint: disable=unused-import @@ -102,6 +107,66 @@ def testParseSystem(self): parser_chain = self._PluginNameToParserChain('windows_services') self.assertEqual(parser_chains.get(parser_chain, 0), 831) + @shared_test_lib.skipUnlessHasTestFile(['artifacts']) + @shared_test_lib.skipUnlessHasTestFile(['SYSTEM']) + def testParseSystemWithArtifactFilters(self): + """Tests the Parse function on a SYSTEM file with artifact filters.""" + parser = winreg.WinRegistryParser() + knowledge_base = knowledge_base_engine.KnowledgeBase() + + artifacts_filters = ['TestRegistryKey', 'TestRegistryValue'] + registry = artifacts_registry.ArtifactDefinitionsRegistry() + reader = artifacts_reader.YamlArtifactsReader() + + registry.ReadFromDirectory(reader, self._GetTestFilePath(['artifacts'])) + + test_filter_file = artifact_filters.ArtifactDefinitionsFilterHelper( + registry, artifacts_filters, knowledge_base) + + test_filter_file.BuildFindSpecs(environment_variables=None) + + find_specs = { + test_filter_file.KNOWLEDGE_BASE_VALUE : knowledge_base.GetValue( + test_filter_file.KNOWLEDGE_BASE_VALUE)} + storage_writer = self._ParseFile( + ['SYSTEM'], parser, knowledge_base_values=find_specs) + + events = list(storage_writer.GetEvents()) + + parser_chains = self._GetParserChains(events) + + # Check the existence of few known plugins, see if they + # are being properly picked up and are parsed. + plugin_names = [ + 'windows_usbstor_devices', 'windows_boot_execute', + 'windows_services'] + for plugin in plugin_names: + expected_parser_chain = self._PluginNameToParserChain(plugin) + self.assertTrue( + expected_parser_chain in parser_chains, + 'Chain {0:s} not found in events.'.format(expected_parser_chain)) + + # Check that the number of events produced by each plugin are correct. + + # There will be 5 usbstor chains for currentcontrolset: + # 'HKEY_LOCAL_MACHINE\System\CurrentControlSet\Enum\USBSTOR' + parser_chain = self._PluginNameToParserChain('windows_usbstor_devices') + self.assertEqual(parser_chains.get(parser_chain, 0), 5) + + # There will be 4 Windows boot execute chains for key_value pairs: + # {key: 'HKEY_LOCAL_MACHINE\System\ControlSet001\Control\Session Manager', + # value: 'BootExecute'} + # {key: 'HKEY_LOCAL_MACHINE\System\ControlSet002\Control\Session Manager', + # value: 'BootExecute'} + parser_chain = self._PluginNameToParserChain('windows_boot_execute') + self.assertEqual(parser_chains.get(parser_chain, 0), 4) + + # There will be 831 windows services chains for keys: + # 'HKEY_LOCAL_MACHINE\System\ControlSet001\services\**' + # 'HKEY_LOCAL_MACHINE\System\ControlSet002\services\**' + parser_chain = self._PluginNameToParserChain('windows_services') + self.assertEqual(parser_chains.get(parser_chain, 0), 831) + if __name__ == '__main__': unittest.main()