Disable mans import task

api_client/python/timesketch_api_client/sketch.py

@@ -464,7 +464,7 @@ def list_timelines(self):
         return timelines
 
     def upload(self, timeline_name, file_path, index=None):
-        """Upload a CSV, JSONL, mans, or Plaso file to the server for indexing.
+        """Upload a CSV, JSONL or Plaso file to the server for indexing.
 
         Args:
             timeline_name: Name of the resulting timeline.

config/dpkg/timesketch-server.timesketch.default

@@ -113,13 +113,13 @@ GOOGLE_OIDC_HOSTED_DOMAIN = None
 GOOGLE_OIDC_USER_WHITELIST = []
 
 #-------------------------------------------------------------------------------
-# Upload and processing of Plaso storage or mans files.
+# Upload and processing of Plaso storage files.
 
 # To enable this feature you need to configure an upload directory and
 # how to reach the Redis database used by the distributed task queue.
 UPLOAD_ENABLED = False
 
-# Folder for temporarily storage of Plaso dump or mans files before being processed and
+# Folder for temporarily storage of Plaso dump before being processed and
 # inserted into the datastore.
 UPLOAD_FOLDER = '/tmp'
 

contrib/timesketch-importer.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # This script watches a directory for new files and executes the importer
-# on any .plaso, .mans, .csv and .jsonl files. To be place in /usr/local/bin/
+# on any .plaso, .csv and .jsonl files. To be place in /usr/local/bin/
 
 config_file="/etc/timesketch-importer.conf"
 if [ -s "$config_file" ]; then

importer_client/python/timesketch_import_client/importer.py

@@ -504,7 +504,7 @@ def add_excel_file(self, filepath, **kwargs):
         self.add_data_frame(data_frame)
 
     def add_file(self, filepath, delimiter=','):
-        """Add a CSV, JSONL, mans or a PLASO file to the buffer.
+        """Add a CSV, JSONL or a PLASO file to the buffer.
 
         Args:
             filepath: the path to the file to add.
@@ -535,7 +535,7 @@ def add_file(self, filepath, delimiter=','):
                         fh, delimiter=delimiter,
                         chunksize=self._threshold_entry):
                     self.add_data_frame(chunk_frame, part_of_iter=True)
-        elif file_ending in ('plaso', 'mans'):
+        elif file_ending == 'plaso':
             self._upload_binary_file(filepath)
 
         elif file_ending == 'jsonl':

importer_client/python/tools/timesketch_importer.py

@@ -51,10 +51,10 @@ def upload_file(
         return 'Sketch needs to be set'
 
     _, _, file_extension = file_path.rpartition('.')
-    if file_extension.lower() not in ('plaso', 'mans', 'csv', 'jsonl'):
+    if file_extension.lower() not in ('plaso', 'csv', 'jsonl'):
         return (
             'File needs to have one of the following extensions: '
-            '.plaso, .mans, .csv, '
+            '.plaso, .csv, '
             '.jsonl (not {0:s})').format(file_extension.lower())
 
     with importer.ImportStreamer() as streamer:

requirements.txt

@@ -31,5 +31,4 @@ Werkzeug==0.16.0
 WTForms==2.2.1
 xlrd==1.2.0
 tabulate==0.8.6
-markdown==3.2.2
-mans_to_es==1.6
+markdown==3.2.2

timesketch/frontend/src/views/SketchManageTimelines.vue

@@ -48,7 +48,7 @@ limitations under the License.
 
             <b-message type="is-success">
               <p>
-                Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, mans, JSONL, or a CSV file.
+                Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, JSONL, or a CSV file.
                 <br>
                 If you are uploading a CSV or JSONL file make sure to read the <a href="https://github.com/google/timesketch/blob/master/docs/Users-Guide.md#adding-timelines" rel="noreferrer" target="_blank">documentation</a> to learn what columns are needed.
               </p>

timesketch/frontend/src/views/SketchOverview.vue

@@ -72,7 +72,7 @@ limitations under the License.
         <div class="card-content">
           <div class="content">
             <p>
-              Supported formats are Plaso storage file, mans, JSONL, or a CSV file.
+              Supported formats are Plaso storage file, JSONL, or a CSV file.
               If you are uploading a CSV or JSONL file make sure to read the <a href="https://github.com/google/timesketch/blob/master/docs/Users-Guide.md#adding-timelines" rel="noreferrer" target="_blank">documentation</a> to learn what columns are needed.
             </p>
             <ts-upload-timeline-form @toggleModal="showUploadTimelineModal = !showUploadTimelineModal"></ts-upload-timeline-form>

timesketch/lib/tasks.py

@@ -29,7 +29,9 @@
 from flask import current_app
 from sqlalchemy import create_engine
 from elasticsearch.exceptions import RequestError
-from mans_to_es import MansToEs
+
+# Disabled until the project can provide a non-ES native import.
+# from mans_to_es import MansToEs
 
 from timesketch import create_celery_app
 from timesketch.lib import errors
@@ -109,10 +111,11 @@ def _get_index_task_class(file_extension):
     """
     if file_extension == 'plaso':
         index_class = run_plaso
-    elif file_extension == 'mans':
-        index_class = run_mans
     elif file_extension in ['csv', 'jsonl']:
         index_class = run_csv_jsonl
+    # Disabled
+    # elif file_extension == 'mans':
+    #    index_class = run_mans
     else:
         raise KeyError('No task that supports {0:s}'.format(file_extension))
     return index_class
@@ -524,20 +527,11 @@ def run_csv_jsonl(file_path, events, timeline_name, index_name, source_type):
     return index_name
 
 
-@celery.task(track_started=True, base=SqlAlchemyTask)
+# Disabled until mans_to_es can produce a stream of events instead of doing
+# raw Elasticsearch operations.
+# pylint: disable=pointless-string-statement
+"""
 def run_mans(file_path, events, timeline_name, index_name, source_type):
-    """Create a Celery task for processing mans file.
-
-    Args:
-        file_path: Path to the mans file.
-        events: A string with the events. Not used in mans.
-        timeline_name: Name of the Timesketch timeline.
-        index_name: Name of the datastore index.
-        source_type: Type of file, csv or jsonl.
-
-    Returns:
-        Name (str) of the index.
-    """
     # Log information to Celery
     message = 'Index timeline [{0:s}] to index [{1:s}] (source: {2:s})'
     logging.info(message.format(timeline_name, index_name, source_type))
@@ -559,3 +553,4 @@ def run_mans(file_path, events, timeline_name, index_name, source_type):
     _set_timeline_status(index_name, status='ready')
 
     return index_name
+"""

timesketch/templates/sketch/timelines.html

@@ -12,7 +12,7 @@
     {% if sketch.has_permission(current_user, 'write') and upload_enabled %}
         <div class="card">
             <h4>Import timeline</h4>
-            <p>You can upload either a Plaso storage file, mans, JSONL, or a CSV file.<br>
+            <p>You can upload either a Plaso storage file, JSONL, or a CSV file.<br>
                 <i>Supported Plaso version: {{ plaso_version }}</i>
             </p>
             <ts-core-upload sketch-id="{{ sketch.id }}" visible="true" btn-text="'Select file'"></ts-core-upload>

timesketch/tsctl.py

@@ -385,7 +385,7 @@ def run(self, file_path, sketch_id, username, timeline_name):
         extension = extension.lstrip('.')
         filename = os.path.basename(file_path_no_extension)
 
-        supported_extensions = ('plaso', 'mans', 'csv', 'jsonl')
+        supported_extensions = ('plaso', 'csv', 'jsonl')
 
         if not os.path.isfile(file_path):
             sys.exit('No such file: {0:s}'.format(file_path))