Changes to expose data stream extents log2timeline#597

appveyor.yml

@@ -1,4 +1,6 @@
 environment:
+  pypi_token:
+    secure: /FwQrmudDyj+Mu3DaxLEowyvwBaY7x1GRt6gYJrVerEAo4PujrTDfMs9/K6PJSN7KkCL/6LQK2VfTD91bbnUwGMiTjfeItu2+aernJtwLLtoDJ22sHgMiajGMqficrHlOc7uNhFMjQsGa7WiiGGo12c/b7z55dNmU2N0EIc086/Z2G6O+n2+oBeT5SbFu5j5XXkwrd98vnW8hryuZPjLauV1mxc6MMNiv3dOgVL8gtWDjW5xZVJvfOTcYA+7MMLPUbMbqMcXkTSRshqUrX/6mw==
   matrix:
   - DESCRIPTION: "Windows with 32-bit Python 3.10"
     MACHINE_TYPE: "x86"
@@ -35,3 +37,7 @@ test_script:
 
 artifacts:
 - path: dist\*.whl
+
+deploy_script:
+- ps: If ($env:APPVEYOR_REPO_TAG -eq "true" -And $isWindows -And $env:MACHINE_TYPE -eq "x86") {
+    Invoke-Expression "${env:PYTHON}\\python.exe -m twine upload dist/*.whl --username __token__ --password ${env:PYPI_TOKEN} --skip-existing" }

config/dpkg/control

@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20220112), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20220112), libfshfs-python3 (>= 20220113), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

dependencies.ini

@@ -66,7 +66,7 @@ version_property: get_version()
 [pyfshfs]
 dpkg_name: libfshfs-python3
 l2tbinaries_name: libfshfs
-minimum_version: 20210722
+minimum_version: 20220113
 pypi_name: libfshfs-python
 rpm_name: libfshfs-python3
 version_property: get_version()

dfvfs.ini

@@ -11,3 +11,4 @@ description_long: dfVFS, or Digital Forensics Virtual File System, provides read
     of dfVFS is to provide a generic interface for accessing file-system objects,
     for which it uses several back-ends that provide the actual implementation of
     the various storage media types, volume systems and file systems.
+pypi_token: /FwQrmudDyj+Mu3DaxLEowyvwBaY7x1GRt6gYJrVerEAo4PujrTDfMs9/K6PJSN7KkCL/6LQK2VfTD91bbnUwGMiTjfeItu2+aernJtwLLtoDJ22sHgMiajGMqficrHlOc7uNhFMjQsGa7WiiGGo12c/b7z55dNmU2N0EIc086/Z2G6O+n2+oBeT5SbFu5j5XXkwrd98vnW8hryuZPjLauV1mxc6MMNiv3dOgVL8gtWDjW5xZVJvfOTcYA+7MMLPUbMbqMcXkTSRshqUrX/6mw==

dfvfs/vfs/hfs_file_entry.py

@@ -8,6 +8,7 @@
 from dfvfs.lib import errors
 from dfvfs.path import hfs_path_spec
 from dfvfs.vfs import attribute
+from dfvfs.vfs import extent
 from dfvfs.vfs import file_entry
 from dfvfs.vfs import hfs_attribute
 from dfvfs.vfs import hfs_directory
@@ -221,6 +222,30 @@ def size(self):
     """int: size of the file entry in bytes or None if not available."""
     return self._fshfs_file_entry.size
 
+  def GetExtents(self, data_stream_name=''):
+    """Retrieves extents of a specific data stream.
+
+    Returns:
+      list[Extent]: extents of the data stream.
+    """
+    extents = []
+    if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and
+        not data_stream_name):
+      for extent_index in range(self._fshfs_file_entry.number_of_extents):
+        extent_offset, extent_size, extent_flags = (
+            self._fshfs_file_entry.get_extent(extent_index))
+
+        if extent_flags & 0x1:
+          extent_type = definitions.EXTENT_TYPE_SPARSE
+        else:
+          extent_type = definitions.EXTENT_TYPE_DATA
+
+        data_stream_extent = extent.Extent(
+            extent_type=extent_type, offset=extent_offset, size=extent_size)
+        extents.append(data_stream_extent)
+
+    return extents
+
   def GetHFSFileEntry(self):
     """Retrieves the HFS file entry.
 

dfvfs/vfs/tsk_file_entry.py

@@ -709,8 +709,6 @@ def GetExtents(self, data_stream_name=''):
     for pytsk_attribute in self._tsk_file:
       if getattr(pytsk_attribute, 'info', None):
         attribute_type = getattr(pytsk_attribute.info, 'type', None)
-        if attribute_type in self._TSK_INTERNAL_ATTRIBUTE_TYPES:
-          continue
 
         name = getattr(pytsk_attribute.info, 'name', None)
         if name:
@@ -721,10 +719,7 @@ def GetExtents(self, data_stream_name=''):
             raise errors.BackEndError(
                 'pytsk3 returned a non UTF-8 formatted name.')
 
-        # The data stream is returned as a name-less attribute of type
-        # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT.
-        if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and
-            attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and
+        if attribute_type == pytsk3.TSK_FS_ATTR_TYPE_HFS_DATA and (
             not name and not data_stream_name):
           data_pytsk_attribute = pytsk_attribute
           break
@@ -734,6 +729,14 @@ def GetExtents(self, data_stream_name=''):
           data_pytsk_attribute = pytsk_attribute
           break
 
+        # The data stream is returned as a name-less attribute of type
+        # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT.
+        if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and
+            attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and
+            not name and not data_stream_name):
+          data_pytsk_attribute = pytsk_attribute
+          break
+
     extents = []
     if data_pytsk_attribute:
       tsk_file_system = self._file_system.GetFsInfo()

requirements.txt

@@ -8,7 +8,7 @@ libbde-python >= 20140531
 libewf-python >= 20131210
 libfsapfs-python >= 20201107
 libfsext-python >= 20220112
-libfshfs-python >= 20210722
+libfshfs-python >= 20220113
 libfsntfs-python >= 20211229
 libfsxfs-python >= 20210726
 libfvde-python >= 20160719

setup.cfg

@@ -21,7 +21,7 @@ requires = libbde-python3 >= 20140531
            libewf-python3 >= 20131210
            libfsapfs-python3 >= 20201107
            libfsext-python3 >= 20220112
-           libfshfs-python3 >= 20210722
+           libfshfs-python3 >= 20220113
            libfsntfs-python3 >= 20211229
            libfsxfs-python3 >= 20210726
            libfvde-python3 >= 20160719

tests/vfs/hfs_file_entry.py

@@ -55,10 +55,10 @@ def testInitialize(self):
 
   def testAccessTime(self):
     """Test the access_time property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -67,10 +67,10 @@ def testAccessTime(self):
 
   def testAddedTime(self):
     """Test the added_time property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -79,10 +79,10 @@ def testAddedTime(self):
 
   def testChangeTime(self):
     """Test the change_time property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -91,10 +91,10 @@ def testChangeTime(self):
 
   def testCreationTime(self):
     """Test the creation_time property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -103,10 +103,10 @@ def testCreationTime(self):
 
   def testModificationTime(self):
     """Test the modification_time property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -115,10 +115,10 @@ def testModificationTime(self):
 
   def testSize(self):
     """Test the size property."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
 
@@ -127,10 +127,9 @@ def testSize(self):
 
   def testGetAttributes(self):
     """Tests the _GetAttributes function."""
-    test_location = '/a_directory/a_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_FILE,
-        location=test_location, parent=self._raw_path_spec)
+        location='/a_directory/a_file', parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
 
@@ -149,10 +148,10 @@ def testGetAttributes(self):
 
   def testGetStat(self):
     """Tests the _GetStat function."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
@@ -181,10 +180,10 @@ def testGetStat(self):
 
   def testGetStatAttribute(self):
     """Tests the _GetStatAttribute function."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
@@ -201,6 +200,32 @@ def testGetStatAttribute(self):
     self.assertEqual(stat_attribute.size, 22)
     self.assertEqual(stat_attribute.type, stat_attribute.TYPE_FILE)
 
+  def testGetExtents(self):
+    """Tests the GetExtents function."""
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_HFS,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
+        parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 1)
+
+    self.assertEqual(extents[0].extent_type, definitions.EXTENT_TYPE_DATA)
+    self.assertEqual(extents[0].offset, 1134592)
+    self.assertEqual(extents[0].size, 4096)
+
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY,
+        location='/a_directory', parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 0)
+
   def testGetFileEntryByPathSpec(self):
     """Tests the GetFileEntryByPathSpec function."""
     path_spec = path_spec_factory.Factory.NewPathSpec(
@@ -212,10 +237,9 @@ def testGetFileEntryByPathSpec(self):
 
   def testGetLinkedFileEntry(self):
     """Tests the GetLinkedFileEntry function."""
-    test_location = '/a_link'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_LINK,
-        location=test_location, parent=self._raw_path_spec)
+        location='/a_link', parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
 
@@ -227,10 +251,10 @@ def testGetLinkedFileEntry(self):
 
   def testGetParentFileEntry(self):
     """Tests the GetParentFileEntry function."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
@@ -243,10 +267,10 @@ def testGetParentFileEntry(self):
 
   def testIsFunctions(self):
     """Tests the Is? functions."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
@@ -262,10 +286,9 @@ def testIsFunctions(self):
     self.assertFalse(file_entry.IsPipe())
     self.assertFalse(file_entry.IsSocket())
 
-    test_location = '/a_directory'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY,
-        location=test_location, parent=self._raw_path_spec)
+        location='/a_directory', parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
 
@@ -335,10 +358,10 @@ def testSubFileEntries(self):
 
   def testDataStreams(self):
     """Tests the data streams functionality."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
@@ -351,10 +374,9 @@ def testDataStreams(self):
 
     self.assertEqual(data_stream_names, [''])
 
-    test_location = '/a_directory'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS, identifier=self._IDENTIFIER_A_DIRECTORY,
-        location=test_location, parent=self._raw_path_spec)
+        location='/a_directory', parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)
 
@@ -368,10 +390,10 @@ def testDataStreams(self):
 
   def testGetDataStream(self):
     """Tests the GetDataStream function."""
-    test_location = '/a_directory/another_file'
     path_spec = path_spec_factory.Factory.NewPathSpec(
         definitions.TYPE_INDICATOR_HFS,
-        identifier=self._IDENTIFIER_ANOTHER_FILE, location=test_location,
+        identifier=self._IDENTIFIER_ANOTHER_FILE,
+        location='/a_directory/another_file',
         parent=self._raw_path_spec)
     file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
     self.assertIsNotNone(file_entry)

tests/vfs/tsk_file_entry.py

@@ -1093,7 +1093,29 @@ def testSize(self):
     self.assertIsNotNone(file_entry)
     self.assertEqual(file_entry.size, 22)
 
-  # TODO: add tests for GetExtents
+  def testGetExtents(self):
+    """Tests the GetExtents function."""
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_TSK, inode=self._INODE_ANOTHER_FILE,
+        location='/a_directory/another_file', parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 1)
+
+    self.assertEqual(extents[0].extent_type, definitions.EXTENT_TYPE_DATA)
+    self.assertEqual(extents[0].offset, 1134592)
+    self.assertEqual(extents[0].size, 4096)
+
+    path_spec = path_spec_factory.Factory.NewPathSpec(
+        definitions.TYPE_INDICATOR_TSK, inode=self._INODE_A_DIRECTORY,
+        location='/a_directory', parent=self._raw_path_spec)
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    self.assertIsNotNone(file_entry)
+
+    extents = file_entry.GetExtents()
+    self.assertEqual(len(extents), 0)
 
   def testGetFileObject(self):
     """Tests the GetFileObject function."""