Added artifacts filter support for files and Windows Registry #1313

plaso/cli/extraction_tool.py

@@ -75,6 +75,7 @@ def _CreateProcessingConfiguration(self, knowledge_base):
     """
     # TODO: pass preferred_encoding.
     configuration = configurations.ProcessingConfiguration()
+    configuration.artifact_filters = self._artifact_filters
     configuration.credentials = self._credential_configurations
     configuration.debug_output = self._debug_mode
     configuration.event_extraction.text_prepend = self._text_prepend

plaso/cli/helpers/__init__.py

@@ -3,6 +3,7 @@
 
 from plaso.cli.helpers import analysis_plugins
 from plaso.cli.helpers import artifact_definitions
+from plaso.cli.helpers import artifact_filters
 from plaso.cli.helpers import data_location
 from plaso.cli.helpers import date_filters
 from plaso.cli.helpers import dynamic_output

plaso/cli/helpers/artifact_definitions.py

@@ -43,6 +43,15 @@ def AddArguments(cls, argument_group):
             'quickly collect data of interest, such as specific files or '
             'Windows Registry keys.'))
 
+    argument_group.add_argument(
+        '--custom_artifact_definitions', '--custom-artifact-definitions',
+        dest='custom_artifact_definitions_path', type=str, metavar='PATH',
+        action='store', help=(
+            'Path to a file containing custom artifact definitions, which are '
+            '.yaml files. Artifact definitions can be used to describe and '
+            'quickly collect data of interest, such as specific files or '
+            'Windows Registry keys.'))
+
   @classmethod
   def ParseOptions(cls, options, configuration_object):
     """Parses and validates options.
@@ -86,6 +95,13 @@ def ParseOptions(cls, options, configuration_object):
       raise errors.BadConfigOption(
           'Unable to determine path to artifact definitions.')
 
+    custom_artifacts_path = getattr(
+        options, 'custom_artifact_definitions_path', None)
+
+    if custom_artifacts_path and not os.path.isfile(custom_artifacts_path):
+      raise errors.BadConfigOption(
+          'No such artifacts filter file: {0:s}.'.format(custom_artifacts_path))
+
     registry = artifacts_registry.ArtifactDefinitionsRegistry()
     reader = artifacts_reader.YamlArtifactsReader()
 
@@ -102,7 +118,18 @@ def ParseOptions(cls, options, configuration_object):
         raise errors.BadConfigOption(
             'Missing required artifact definition: {0:s}'.format(name))
 
-    setattr(configuration_object, '_artifacts_registry', registry)
+    if custom_artifacts_path:
+      try:
+        registry.ReadFromFile(reader, custom_artifacts_path)
+
+      except (KeyError, artifacts_errors.FormatError) as exception:
+        raise errors.BadConfigOption((
+            'Unable to read artifact definitions from: {0:s} with error: '
+            '{1!s}').format(custom_artifacts_path, exception))
+
+    setattr(configuration_object, '_artifact_definitions_path', artifacts_path)
+    setattr(
+        configuration_object, '_custom_artifacts_path', custom_artifacts_path)
 
 
 manager.ArgumentHelperManager.RegisterHelper(ArtifactDefinitionsArgumentsHelper)

plaso/cli/helpers/artifact_filters.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+"""The artifacts filter file CLI arguments helper."""
+
+from __future__ import unicode_literals
+
+import os
+
+from plaso.cli import tools
+from plaso.cli.helpers import interface
+from plaso.cli.helpers import manager
+from plaso.lib import errors
+
+
+class ArtifactFiltersArgumentsHelper(interface.ArgumentsHelper):
+  """Artifacts filter file CLI arguments helper."""
+
+  NAME = 'artifact_filters'
+  DESCRIPTION = 'Artifact filters command line arguments.'
+
+  @classmethod
+  def AddArguments(cls, argument_group):
+    """Adds command line arguments to an argument group.
+
+    This function takes an argument parser or an argument group object and adds
+    to it all the command line arguments this helper supports.
+
+    Args:
+      argument_group (argparse._ArgumentGroup|argparse.ArgumentParser):
+          argparse group.
+    """
+    argument_group.add_argument(
+        '--artifact_filters', '--artifact-filters',
+        dest='artifact_filters', type=str, default=None,
+        action='store', help=(
+            'Names of forensic artifact definitions, provided on the command '
+            'command line (comma separated). Forensic artifacts are stored '
+            'in .yaml files that are directly pulled from the artifact '
+            'definitions project. You can also specify a custom '
+            'artifacts yaml file (see --custom_artifact_definitions). Artifact '
+            'definitions can be used to describe and quickly collect data of '
+            'interest, such as specific files or Windows Registry keys.'))
+
+    argument_group.add_argument(
+        '--artifact_filters_file', '--artifact-filters_file',
+        dest='artifact_filters', type=str, default=None,
+        action='store', help=(
+            'Names of forensic artifact definitions, provided in a file with '
+            'one artifact name per line. Forensic artifacts are stored in '
+            '.yaml files that are directly pulled from the artifact '
+            'definitions project. You can also specify a custom artifacts '
+            'yaml file (see --custom_artifact_definitions). Artifact '
+            'definitions can be used to describe and quickly collect data of '
+            'interest, such as specific files or Windows Registry keys.'))
+
+  @classmethod
+  def ParseOptions(cls, options, configuration_object):
+    """Parses and validates options.
+
+    Args:
+      options (argparse.Namespace): parser options.
+      configuration_object (CLITool): object to be configured by the argument
+          helper.
+
+    Raises:
+      BadConfigObject: when the configuration object is of the wrong type.
+      BadConfigOption: if the required artifact definitions are not defined.
+    """
+    if not isinstance(configuration_object, tools.CLITool):
+      raise errors.BadConfigObject(
+          'Configuration object is not an instance of CLITool')
+
+    artifact_filters = cls._ParseStringOption(options, 'artifact_filters')
+    artifact_filters_file = cls._ParseStringOption(
+        options, 'artifact_filters_file')
+    filter_file = cls._ParseStringOption(options, 'file_filter')
+
+    if artifact_filters and artifact_filters_file:
+      raise errors.BadConfigOption(
+          'Please only specify artifact definition names in a file '
+          'or on the command line.')
+
+    if (artifact_filters_file or artifact_filters) and filter_file:
+      raise errors.BadConfigOption(
+          'Please do not specify both artifact definitions and legacy filters.')
+
+    if artifact_filters_file and os.path.isfile(artifact_filters_file):
+      with open(artifact_filters_file) as file_object:
+        file_content = file_object.read()
+        artifact_filters = file_content.splitlines()
+    elif artifact_filters:
+      artifact_filters = [name.strip() for name in artifact_filters.split(',')]
+
+    setattr(configuration_object, '_artifact_filters', artifact_filters)
+
+
+manager.ArgumentHelperManager.RegisterHelper(ArtifactFiltersArgumentsHelper)

plaso/cli/image_export_tool.py

@@ -17,8 +17,8 @@
 from plaso.cli import logger
 from plaso.cli import storage_media_tool
 from plaso.cli.helpers import manager as helpers_manager
+from plaso.engine import engine
 from plaso.engine import extractors
-from plaso.engine import filter_file
 from plaso.engine import knowledge_base
 from plaso.engine import path_helper
 from plaso.filters import file_entry as file_entry_filters
@@ -73,7 +73,10 @@ def __init__(self, input_reader=None, output_writer=None):
     super(ImageExportTool, self).__init__(
         input_reader=input_reader, output_writer=output_writer)
     self._abort = False
+    self._artifact_definitions_path = None
+    self._artifact_filters = None
     self._artifacts_registry = None
+    self._custom_artifacts_path = None
     self._destination_path = None
     self._digests = {}
     self._filter_collection = file_entry_filters.FileEntryFilterCollection()
@@ -287,7 +290,8 @@ def _ExtractFileEntry(
   # TODO: merge with collector and/or engine.
   def _ExtractWithFilter(
       self, source_path_specs, destination_path, output_writer,
-      filter_file_path, skip_duplicates=True):
+      artifact_filters, filter_file, artifact_definitions_path,
+      custom_artifacts_path, skip_duplicates=True):
     """Extracts files using a filter expression.
 
     This method runs the file extraction process on the image and
@@ -297,8 +301,12 @@ def _ExtractWithFilter(
       source_path_specs (list[dfvfs.PathSpec]): path specifications to extract.
       destination_path (str): path where the extracted files should be stored.
       output_writer (CLIOutputWriter): output writer.
-      filter_file_path (str): path of the file that contains the filter
-          expressions.
+      artifact_definitions_path (str): path to artifact definitions file.
+      custom_artifacts_path (str): path to custom artifact definitions file.
+      artifact_filters (list[str]): names of artifact definitions that are
+          used for filtering file system and Windows Registry key paths.
+      filter_file (str): path of the file that contains the filter file path
+          filters.
       skip_duplicates (Optional[bool]): True if files with duplicate content
           should be skipped.
     """
@@ -314,14 +322,13 @@ def _ExtractWithFilter(
       output_writer.Write(
           'Extracting file entries from: {0:s}\n'.format(display_name))
 
-      environment_variables = self._knowledge_base.GetEnvironmentVariables()
-      filter_file_object = filter_file.FilterFile(filter_file_path)
-      find_specs = filter_file_object.BuildFindSpecs(
-          environment_variables=environment_variables)
+      filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs(
+          artifact_definitions_path, custom_artifacts_path,
+          self._knowledge_base, artifact_filters, filter_file)
 
       searcher = file_system_searcher.FileSystemSearcher(
           file_system, mount_point)
-      for path_spec in searcher.Find(find_specs=find_specs):
+      for path_spec in searcher.Find(find_specs=filter_find_specs):
         self._ExtractFileEntry(
             path_spec, destination_path, output_writer,
             skip_duplicates=skip_duplicates)
@@ -399,7 +406,7 @@ def _ParseFilterOptions(self, options):
     Raises:
       BadConfigOption: if the options are invalid.
     """
-    names = ['date_filters', 'filter_file']
+    names = ['artifact_filters', 'date_filters', 'filter_file']
     helpers_manager.ArgumentHelperManager.ParseOptions(
         options, self, names=names)
 
@@ -416,7 +423,7 @@ def _ParseFilterOptions(self, options):
     except (IOError, ValueError) as exception:
       raise errors.BadConfigOption(exception)
 
-    if self._filter_file:
+    if self._artifact_filters or self._filter_file:
       self.has_filters = True
     else:
       self.has_filters = self._filter_collection.HasFilters()
@@ -559,7 +566,7 @@ def AddFilterOptions(self, argument_group):
     Args:
       argument_group (argparse._ArgumentGroup): argparse argument group.
     """
-    names = ['date_filters', 'filter_file']
+    names = ['artifact_filters', 'date_filters', 'filter_file']
     helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
         argument_group, names=names)
 
@@ -749,10 +756,12 @@ def ProcessSources(self):
     if not os.path.isdir(self._destination_path):
       os.makedirs(self._destination_path)
 
-    if self._filter_file:
+    if self._artifact_filters or self._filter_file:
       self._ExtractWithFilter(
           self._source_path_specs, self._destination_path, self._output_writer,
-          self._filter_file, skip_duplicates=self._skip_duplicates)
+          self._artifact_filters, self._filter_file,
+          self._artifact_definitions_path, self._custom_artifacts_path,
+          skip_duplicates=self._skip_duplicates)
     else:
       self._Extract(
           self._source_path_specs, self._destination_path, self._output_writer,

plaso/cli/log2timeline_tool.py

@@ -23,7 +23,6 @@
 from plaso.cli import views
 from plaso.cli.helpers import manager as helpers_manager
 from plaso.engine import engine
-from plaso.engine import filter_file
 from plaso.engine import single_process as single_process_engine
 from plaso.lib import definitions
 from plaso.lib import errors
@@ -167,7 +166,8 @@ def ParseArguments(self):
         'extraction arguments')
 
     argument_helper_names = [
-        'extraction', 'filter_file', 'hashers', 'parsers', 'yara_rules']
+        'artifact_filters', 'extraction', 'filter_file', 'hashers',
+        'parsers', 'yara_rules']
     helpers_manager.ArgumentHelperManager.AddCommandLineArguments(
         extraction_group, names=argument_helper_names)
 
@@ -317,8 +317,9 @@ def ParseOptions(self, options):
     self._ParseInformationalOptions(options)
 
     argument_helper_names = [
-        'artifact_definitions', 'extraction', 'filter_file', 'status_view',
-        'storage_file', 'storage_format', 'text_prepend', 'yara_rules']
+        'artifact_definitions', 'artifact_filters', 'extraction',
+        'filter_file', 'status_view', 'storage_file', 'storage_format',
+        'text_prepend', 'yara_rules']
     helpers_manager.ArgumentHelperManager.ParseOptions(
         options, self, names=argument_helper_names)
 
@@ -370,7 +371,9 @@ def ExtractEventsFromSources(self):
 
     self._status_view.SetMode(self._status_view_mode)
     self._status_view.SetSourceInformation(
-        self._source_path, self._source_type, filter_file=self._filter_file)
+        self._source_path, self._source_type,
+        artifact_filters=self._artifact_filters,
+        filter_file=self._filter_file)
 
     status_update_callback = (
         self._status_view.GetExtractionStatusUpdateCallback())
@@ -380,9 +383,10 @@ def ExtractEventsFromSources(self):
     self._output_writer.Write('Processing started.\n')
 
     session = engine.BaseEngine.CreateSession(
+        artifact_filter_names=self._artifact_filters,
         command_line_arguments=self._command_line_arguments,
         debug_mode=self._debug_mode,
-        filter_file=self._filter_file,
+        filter_file_path=self._filter_file,
         preferred_encoding=self.preferred_encoding,
         preferred_time_zone=self._preferred_time_zone,
         preferred_year=self._preferred_year)
@@ -415,13 +419,15 @@ def ExtractEventsFromSources(self):
     self._SetExtractionParsersAndPlugins(configuration, session)
     self._SetExtractionPreferredTimeZone(extraction_engine.knowledge_base)
 
-    filter_find_specs = None
-    if configuration.filter_file:
-      environment_variables = (
-          extraction_engine.knowledge_base.GetEnvironmentVariables())
-      filter_file_object = filter_file.FilterFile(configuration.filter_file)
-      filter_find_specs = filter_file_object.BuildFindSpecs(
-          environment_variables=environment_variables)
+    artifact_filters = getattr(configuration, '_artifact_filters', None)
+    artifact_definitions_path = getattr(
+        configuration, '_artifact_definitions_path', None)
+    custom_artifacts_path = getattr(
+        configuration, '_custom_artifacts_path', None)
+    filter_find_specs = engine.BaseEngine.BuildFilterFindSpecs(
+        artifact_definitions_path, custom_artifacts_path,
+        extraction_engine.knowledge_base, artifact_filters,
+        configuration.filter_file)
 
     processing_status = None
     if single_process_mode:

plaso/cli/pinfo_tool.py

@@ -353,6 +353,7 @@ def _PrintSessionsDetails(self, storage):
       command_line_arguments = session.command_line_arguments or 'N/A'
       parser_filter_expression = session.parser_filter_expression or 'N/A'
       preferred_encoding = session.preferred_encoding or 'N/A'
+      artifact_filters = session.artifact_filters or 'N/A'
       filter_file = session.filter_file or 'N/A'
 
       title = 'Session: {0!s}'.format(session_identifier)
@@ -368,6 +369,7 @@ def _PrintSessionsDetails(self, storage):
       table_view.AddRow(['Enabled parser and plugins', enabled_parser_names])
       table_view.AddRow(['Preferred encoding', preferred_encoding])
       table_view.AddRow(['Debug mode', session.debug_mode])
+      table_view.AddRow(['Artifact filters', artifact_filters])
       table_view.AddRow(['Filter file', filter_file])
 
       table_view.Write(self._output_writer)