Merge branch 'release/v1.3' into 'master'

Release/v1.3

See merge request cert/malduck!14

docs/compression.rst

@@ -7,3 +7,5 @@ Compression algorithms
     :members:
 .. autoclass:: malduck.compression.gzip.Gzip
     :members:
+.. autoclass:: malduck.compression.lznt1.Lznt1
+    :members:

docs/conf.py

@@ -22,7 +22,7 @@
 author = 'CERT Polska'
 
 # The full version, including alpha/beta/rc tags
-release = '1.2.0'
+release = '1.3.0'
 
 
 # -- General configuration ---------------------------------------------------

malduck/__init__.py

@@ -20,7 +20,7 @@
 
 from .short import (
     aes, blowfish, des3, rc4, pe, aplib, gzip, procmem, procmempe, cuckoomem, pad, unpad,
-    insn, rsa, verify, base64, rabbit, serpent
+    insn, rsa, verify, base64, rabbit, serpent, lznt1
 )
 
 from .string.bin import (

malduck/compression/components/lznt1.py

@@ -0,0 +1,117 @@
+# Rekall Memory Forensics
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Author: Michael Cohen [email protected].
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+
+"""Decompression support for the LZNT1 compression algorithm.
+
+Reference:
+http://msdn.microsoft.com/en-us/library/jj665697.aspx
+(2.5 LZNT1 Algorithm Details)
+
+https://github.com/libyal/reviveit/
+https://github.com/sleuthkit/sleuthkit/blob/develop/tsk/fs/ntfs.c
+"""
+import array
+from io import BytesIO
+
+import struct
+
+__all__ = ['decompress_data']
+
+
+def get_displacement(offset):
+    """Calculate the displacement."""
+    result = 0
+    while offset >= 0x10:
+        offset >>= 1
+        result += 1
+
+    return result
+
+
+DISPLACEMENT_TABLE = array.array(
+    'B', [get_displacement(x) for x in range(8192)])
+
+COMPRESSED_MASK = 1 << 15
+SIGNATURE_MASK = 3 << 12
+SIZE_MASK = (1 << 12) - 1
+TAG_MASKS = [(1 << i) for i in range(0, 8)]
+
+
+def decompress_data(cdata):
+    """Decompresses the data."""
+
+    in_fd = BytesIO(cdata)
+    output_fd = BytesIO()
+    block_end = 0
+
+    while in_fd.tell() < len(cdata):
+        block_offset = in_fd.tell()
+        uncompressed_chunk_offset = output_fd.tell()
+
+        block_header = struct.unpack("<H", in_fd.read(2))[0]
+
+        if block_header & SIGNATURE_MASK != SIGNATURE_MASK:
+            break
+
+        size = (block_header & SIZE_MASK)
+
+        block_end = block_offset + size + 3
+
+        if block_header & COMPRESSED_MASK:
+            while in_fd.tell() < block_end:
+                header = ord(in_fd.read(1))
+
+                for mask in TAG_MASKS:
+                    if in_fd.tell() >= block_end:
+                        break
+
+                    if header & mask:
+                        pointer = struct.unpack("<H", in_fd.read(2))[0]
+                        displacement = DISPLACEMENT_TABLE[
+                            output_fd.tell() - uncompressed_chunk_offset - 1]
+
+                        symbol_offset = (pointer >> (12 - displacement)) + 1
+                        symbol_length = (pointer & (0xFFF >> displacement)) + 3
+
+                        output_fd.seek(-symbol_offset, 2)
+                        data = output_fd.read(symbol_length)
+
+                        # Pad the data to make it fit.
+                        if 0 < len(data) < symbol_length:
+                            data = data * (symbol_length // len(data) + 1)
+                            data = data[:symbol_length]
+
+                        output_fd.seek(0, 2)
+
+                        output_fd.write(data)
+
+                    else:
+                        data = in_fd.read(1)
+
+                        output_fd.write(data)
+
+        else:
+            # Block is not compressed
+            data = in_fd.read(size + 1)
+            output_fd.write(data)
+
+    result = output_fd.getvalue()
+
+    return result

malduck/compression/lznt1.py

@@ -0,0 +1,11 @@
+from .components.lznt1 import decompress_data
+
+
+class Lznt1(object):
+    """
+    Implementation of LZNT1 decompression. Allows to decompress data compressed by RtlCompressBuffer
+    """
+    def decompress(self, buf):
+        return decompress_data(buf)
+
+    __call__ = decompress

malduck/main.py

@@ -4,8 +4,9 @@
 
 import codecs
 import click
+import os
 
-from .procmem import CuckooProcessMemory
+from .procmem import CuckooProcessMemory, ProcessMemoryPE
 from .py2compat import ensure_string
 
 
@@ -21,3 +22,20 @@ def cuckoomem_list(mempath):
         for region in p.regions:
             print("0x%08x .. 0x%08x %s" % (region.addr, region.addr + region.size,
                                            ensure_string(codecs.escape_encode(p.readv(region.addr, 16))[0])))
+
+
+@main.command("fixpe")
+@click.argument("mempath", type=click.Path(exists=True))
+@click.argument("outpath", type=click.Path(), required=False)
+@click.option("--force/--no-force", "-f", default=False, help="Try to fix dump even if it's correctly parsed as PE")
+def fixpe(mempath, outpath, force):
+    with ProcessMemoryPE.from_file(mempath) as p:
+        if not force and p.is_image_loaded_as_memdump():
+            click.echo("Input file looks like correct PE file. Use -f if you want to fix it anyway.")
+            return 1
+        outpath = outpath or mempath + ".exe"
+        if not force and os.path.isfile(outpath):
+            click.confirm("{} exists. Overwrite?".format(outpath), abort=True)
+        with open(outpath, "wb") as f:
+            f.write(p.store())
+        click.echo("Fixed {} => {}".format(mempath, outpath))

malduck/pe.py

@@ -40,7 +40,10 @@ def __getitem__(self, item):
         return self.memory.readv(start, stop - start)
 
     def find(self, str, beg=0, end=None):
-        return next(self.memory.regexv(str, self.memory.imgbase + beg, end-beg))
+        try:
+            return next(self.memory.regexv(str, self.memory.imgbase + beg, end and end - beg))
+        except StopIteration:
+            return -1
 
 
 class PE(object):
@@ -109,6 +112,104 @@ def section(self, name):
             if section.Name.rstrip(b"\x00") == ensure_bytes(name):
                 return section
 
+    def directory(self, name):
+        """
+        Get pefile directory entry by identifier
+
+        :param name: shortened pefile directory entry identifier (e.g. 'IMPORT' for 'IMAGE_DIRECTORY_ENTRY_IMPORT')
+        :rtype: :class:`pefile.Structure`
+        """
+        return self.optional_header.DATA_DIRECTORY[
+            pefile.DIRECTORY_ENTRY.get('IMAGE_DIRECTORY_ENTRY_'+name)
+        ]
+
+    def structure(self, rva, format):
+        """
+        Get internal pefile Structure from specified rva
+
+        :param format: :class:`pefile.Structure` format
+                       (e.g. :py:attr:`pefile.PE.__IMAGE_LOAD_CONFIG_DIRECTORY64_format__`)
+        :rtype: :class:`pefile.Structure`
+        """
+        structure = pefile.Structure(format)
+        structure.__unpack__(self.pe.get_data(rva, structure.sizeof()))
+        return structure
+
+    def validate_import_names(self):
+        """
+        Returns True if the first 8 imported library entries have valid library names
+        """
+        import_dir = self.directory('IMPORT')
+        if not import_dir.VirtualAddress:
+            # There's nothing wrong with no imports
+            return True
+        try:
+            import_rva = import_dir.VirtualAddress
+            # Don't go further than 8 entries
+            for _ in range(8):
+                import_desc = self.structure(
+                    import_rva,
+                    pefile.PE.__IMAGE_IMPORT_DESCRIPTOR_format__)
+                if import_desc.all_zeroes():
+                    # End of import-table
+                    break
+                import_dllname = self.pe.get_string_at_rva(import_desc.Name, pefile.MAX_DLL_LENGTH)
+                if not pefile.is_valid_dos_filename(import_dllname):
+                    # Invalid import filename found
+                    return False
+                import_rva += import_desc.sizeof()
+            return True
+        except pefile.PEFormatError:
+            return False
+
+    def validate_resources(self):
+        """
+        Returns True if first level of resource tree looks consistent
+        """
+        resource_dir = self.directory('RESOURCE')
+        if not resource_dir.VirtualAddress:
+            # There's nothing wrong with no resources
+            return True
+        try:
+            resource_rva = resource_dir.VirtualAddress
+            resource_desc = self.structure(
+                resource_rva,
+                pefile.PE.__IMAGE_RESOURCE_DIRECTORY_format__)
+            resource_no = resource_desc.NumberOfNamedEntries + resource_desc.NumberOfIdEntries
+            if not 0 <= resource_no < 128:
+                # Incorrect resource number
+                return False
+            for rsrc_idx in range(resource_no):
+                resource_entry_desc = self.structure(
+                    resource_rva + resource_desc.sizeof() + rsrc_idx * 8,
+                    pefile.PE.__IMAGE_RESOURCE_DIRECTORY_ENTRY_format__
+                )
+                if self.pe.get_word_at_rva(resource_rva + resource_entry_desc.OffsetToData & 0x7fffffff) is None:
+                    return False
+            return True
+        except pefile.PEFormatError:
+            return False
+
+    def validate_padding(self):
+        """
+        Returns True if area between first non-bss section and first 4kB doesn't have only null-bytes
+        """
+        section_start_offs = None
+        for section in self.sections:
+            if section.SizeOfRawData > 0:
+                section_start_offs = section.PointerToRawData
+        if section_start_offs is None:
+            # No non-bss sections? Is it real PE file?
+            return False
+        if section_start_offs > 0x1000:
+            # Unusual - try checking last 512 bytes
+            section_start_offs = 0x800
+        try:
+            data_len = 0x1000 - section_start_offs
+            return self.pe.get_data(section_start_offs, data_len) != b"\x00" * data_len
+        except pefile.PEFormatError:
+            return False
+
     def resources(self, name):
         """
         Finds resource objects by specified name or type