Changes to expose data stream extents #597 (#630)

dfvfs/lib/definitions.py

@@ -26,6 +26,11 @@
 ENCRYPTION_MODE_ECB = 'ecb'
 ENCRYPTION_MODE_OFB = 'ofb'
 
+# The extent types.
+EXTENT_TYPE_COMPRESSED = 'compressed'
+EXTENT_TYPE_DATA = 'data'
+EXTENT_TYPE_SPARSE = 'sparse'
+
 # The type indicator definitions.
 TYPE_INDICATOR_APFS = 'APFS'
 TYPE_INDICATOR_APFS_CONTAINER = 'APFS_CONTAINER'

dfvfs/vfs/extent.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""The Virtual File System (VFS) extent."""
+
+
+class Extent(object):
+  """Extent.
+
+  Attributes:
+    extent_type (str): type of the extent, for example EXTENT_TYPE_SPARSE.
+    offset (int): offset of the extent relative from the start of the file
+        system in bytes.
+    size (int): size of the extent in bytes.
+  """
+
+  def __init__(self, extent_type=None, offset=None, size=None):
+    """Initializes an extent.
+
+    Args:
+      extent_type (Optional[str]): type of the extent, for example
+          EXTENT_TYPE_SPARSE.
+      offset (Optional[int]): offset of the extent relative from the start of
+          the file system in bytes.
+      size (Optional{int]): size of the extent in bytes.
+    """
+    super(Extent, self).__init__()
+    self.extent_type = extent_type
+    self.offset = offset
+    self.size = size

dfvfs/vfs/fake_file_entry.py

@@ -114,7 +114,7 @@ def size(self):
     return size
 
   def GetFileObject(self, data_stream_name=''):
-    """Retrieves the file-like object.
+    """Retrieves a file-like object of a specific data stream.
 
     Args:
       data_stream_name (Optional[str]): name of the data stream, where an empty

dfvfs/vfs/file_entry.py

@@ -308,8 +308,16 @@ def GetDataStream(self, name, case_sensitive=True):
 
     return matching_data_stream
 
+  def GetExtents(self, data_stream_name=''):  # pylint: disable=unused-argument
+    """Retrieves extents of a specific data stream.
+
+    Returns:
+      list[Extent]: extents of the data stream.
+    """
+    return []
+
   def GetFileObject(self, data_stream_name=''):
-    """Retrieves the file-like object.
+    """Retrieves a file-like object of a specific data stream.
 
     Args:
       data_stream_name (Optional[str]): name of the data stream, where an empty

dfvfs/vfs/ntfs_file_entry.py

@@ -13,6 +13,7 @@
 from dfvfs.path import ntfs_path_spec
 from dfvfs.resolver import resolver
 from dfvfs.vfs import attribute
+from dfvfs.vfs import extent
 from dfvfs.vfs import file_entry
 from dfvfs.vfs import ntfs_attribute
 from dfvfs.vfs import ntfs_data_stream
@@ -231,8 +232,50 @@ def size(self):
     """int: size of the file entry in bytes or None if not available."""
     return getattr(self._fsntfs_file_entry, 'size', None)
 
+  def GetExtents(self, data_stream_name=''):
+    """Retrieves extents of a specific data stream.
+
+    Returns:
+      list[Extent]: extents of the data stream.
+    """
+    extents = []
+    if data_stream_name:
+      fsntfs_data_stream = (
+          self._fsntfs_file_entry.get_alternate_data_stream_by_name(
+              data_stream_name))
+
+      if fsntfs_data_stream:
+        for extent_index in range(fsntfs_data_stream.number_of_extents):
+          extent_offset, extent_size, extent_flags = (
+              fsntfs_data_stream.get_extent(extent_index))
+
+          if extent_flags & 0x1:
+            extent_type = definitions.EXTENT_TYPE_SPARSE
+          else:
+            extent_type = definitions.EXTENT_TYPE_DATA
+
+          data_stream_extent = extent.Extent(
+              extent_type=extent_type, offset=extent_offset, size=extent_size)
+          extents.append(data_stream_extent)
+
+    else:
+      for extent_index in range(self._fsntfs_file_entry.number_of_extents):
+        extent_offset, extent_size, extent_flags = (
+            self._fsntfs_file_entry.get_extent(extent_index))
+
+        if extent_flags & 0x1:
+          extent_type = definitions.EXTENT_TYPE_SPARSE
+        else:
+          extent_type = definitions.EXTENT_TYPE_DATA
+
+        data_stream_extent = extent.Extent(
+            extent_type=extent_type, offset=extent_offset, size=extent_size)
+        extents.append(data_stream_extent)
+
+    return extents
+
   def GetFileObject(self, data_stream_name=''):
-    """Retrieves the file-like object.
+    """Retrieves a file-like object of a specific data stream.
 
     Args:
       data_stream_name (Optional[str]): data stream name, where an empty

dfvfs/vfs/tsk_file_entry.py

@@ -15,6 +15,7 @@
 from dfvfs.path import tsk_path_spec
 from dfvfs.resolver import resolver
 from dfvfs.vfs import attribute
+from dfvfs.vfs import extent
 from dfvfs.vfs import file_entry
 from dfvfs.vfs import tsk_attribute
 from dfvfs.vfs import tsk_data_stream
@@ -694,8 +695,68 @@ def size(self):
     """int: size of the file entry in bytes or None if not available."""
     return getattr(self._tsk_file.info.meta, 'size', None)
 
+  def GetExtents(self, data_stream_name=''):
+    """Retrieves extents of a specific data stream.
+
+    Returns:
+      list[Extent]: extents of the data stream.
+
+    Raises:
+      BackEndError: if pytsk3 returns a non UTF-8 formatted name or no file
+          system block size.
+    """
+    data_pytsk_attribute = None
+    for pytsk_attribute in self._tsk_file:
+      if getattr(pytsk_attribute, 'info', None):
+        attribute_type = getattr(pytsk_attribute.info, 'type', None)
+        if attribute_type in self._TSK_INTERNAL_ATTRIBUTE_TYPES:
+          continue
+
+        name = getattr(pytsk_attribute.info, 'name', None)
+        if name:
+          try:
+            # pytsk3 returns an UTF-8 encoded byte string.
+            name = name.decode('utf8')
+          except UnicodeError:
+            raise errors.BackEndError(
+                'pytsk3 returned a non UTF-8 formatted name.')
+
+        # The data stream is returned as a name-less attribute of type
+        # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT.
+        if (attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and not name and
+            not data_stream_name):
+          data_pytsk_attribute = pytsk_attribute
+          break
+
+        if attribute_type == pytsk3.TSK_FS_ATTR_TYPE_NTFS_DATA and (
+            (not name and not data_stream_name) or (name == data_stream_name)):
+          data_pytsk_attribute = pytsk_attribute
+          break
+
+    extents = []
+    if data_pytsk_attribute:
+      tsk_file_system = self._file_system.GetFsInfo()
+      block_size = getattr(tsk_file_system.info, 'block_size', None)
+      if not block_size:
+        raise errors.BackEndError('pytsk3 returned no file system block size.')
+
+      for pytsk_attr_run in data_pytsk_attribute:
+        if pytsk_attr_run.flags & pytsk3.TSK_FS_ATTR_RUN_FLAG_SPARSE:
+          extent_type = definitions.EXTENT_TYPE_SPARSE
+        else:
+          extent_type = definitions.EXTENT_TYPE_DATA
+
+        extent_offset = pytsk_attr_run.addr * block_size
+        extent_size = pytsk_attr_run.len * block_size
+
+        data_stream_extent = extent.Extent(
+            extent_type=extent_type, offset=extent_offset, size=extent_size)
+        extents.append(data_stream_extent)
+
+    return extents
+
   def GetFileObject(self, data_stream_name=''):
-    """Retrieves the file-like object.
+    """Retrieves a file-like object of a specific data stream.
 
     Args:
       data_stream_name (Optional[str]): data stream name, where an empty
@@ -721,6 +782,9 @@ def GetFileObject(self, data_stream_name=''):
 
       setattr(path_spec, 'data_stream', data_stream_name)
 
+    if self.entry_type != definitions.FILE_ENTRY_TYPE_FILE:
+      return None
+
     return resolver.Resolver.OpenFileObject(
         path_spec, resolver_context=self._resolver_context)
 

tests/vfs/extent.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Tests for the VFS extent."""
+
+import unittest
+
+from dfvfs.vfs import extent
+
+from tests import test_lib as shared_test_lib
+
+
+class ExtentTest(shared_test_lib.BaseTestCase):
+  """Tests the VFS extent."""
+
+  def testInitialize(self):
+    """Test the __init__ function."""
+    test_extent = extent.Extent()
+    self.assertIsNotNone(test_extent)
+
+
+if __name__ == '__main__':
+  unittest.main()

tests/vfs/fake_file_entry.py

@@ -131,12 +131,14 @@ def testGetFileObject(self):
     self.assertIsNotNone(file_entry)
 
     file_object = file_entry.GetFileObject()
-
     self.assertIsNotNone(file_object)
 
     file_data = file_object.read()
     self.assertEqual(file_data, b'FILE1')
 
+    file_object = file_entry.GetFileObject(data_stream_name='bogus')
+    self.assertIsNone(file_object)
+
   def testGetParentFileEntry(self):
     """Tests the GetParentFileEntry function."""
     test_file = '/test_data/testdir_fake/file1.txt'

tests/vfs/file_entry.py

@@ -217,16 +217,24 @@ def testGetDataStream(self):
     with self.assertRaises(ValueError):
       test_file_entry.GetDataStream(0)
 
+  def testGetExtents(self):
+    """Tests the GetExtents function."""
+    test_file_entry = TestFileEntry(
+        self._resolver_context, self._file_system, self._path_spec)
+
+    extents = test_file_entry.GetExtents()
+    self.assertEqual(len(extents), 0)
+
   def testGetFileObject(self):
     """Tests the GetFileObject function."""
     test_file_entry = TestFileEntry(
         self._resolver_context, self._file_system, self._path_spec)
 
-    file_object = test_file_entry.GetFileObject('bogus')
+    file_object = test_file_entry.GetFileObject(data_stream_name='bogus')
     self.assertIsNone(file_object)
 
     with self.assertRaises(errors.NotSupported):
-      test_file_entry.GetFileObject('')
+      test_file_entry.GetFileObject()
 
   def testGetFileSystem(self):
     """Tests the GetFileSystem function."""

tests/vfs/ntfs_file_entry.py

@@ -169,7 +169,60 @@ def testGetFileEntryByPathSpec(self):
 
     self.assertIsNotNone(file_entry)
 
-  # TODO: add tests for GetFileObject
+  def testGetExtents(self):
+    """Tests the GetExtents function."""
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_NTFS, location='\\$UpCase', mft_entry=10,
+        parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 1)
+
+    self.assertEqual(extents[0].extent_type, definitions.EXTENT_TYPE_DATA)
+    self.assertEqual(extents[0].offset, 823296)
+    self.assertEqual(extents[0].size, 131072)
+
+    extents = file_entry.GetExtents(data_stream_name='$Info')
+    # No extents are returned for data store in the $DATA attribute.
+    self.assertEqual(len(extents), 0)
+
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_NTFS, location='\\a_directory',
+        mft_entry=self._MFT_ENTRY_A_DIRECTORY, parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 0)
+
+  def testGetFileObject(self):
+    """Tests the GetFileObject function."""
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_NTFS, location='\\$UpCase', mft_entry=10,
+        parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    file_object = file_entry.GetFileObject()
+    self.assertIsNotNone(file_object)
+
+    self.assertEqual(file_object.get_size(), 131072)
+
+    file_object = file_entry.GetFileObject(data_stream_name='$Info')
+    self.assertIsNotNone(file_object)
+
+    self.assertEqual(file_object.get_size(), 32)
+
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_NTFS, location='\\a_directory',
+        mft_entry=self._MFT_ENTRY_A_DIRECTORY, parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    file_object = file_entry.GetFileObject()
+    self.assertIsNone(file_object)
 
   def testGetLinkedFileEntry(self):
     """Tests the GetLinkedFileEntry function."""
@@ -186,7 +239,6 @@ def testGetParentFileEntry(self):
     self.assertIsNotNone(file_entry)
 
     parent_file_entry = file_entry.GetParentFileEntry()
-
     self.assertIsNotNone(parent_file_entry)
 
     self.assertEqual(parent_file_entry.name, 'a_directory')