* Implemented pe plugins for viewing a pe file in the console.

* Updated the Type generator to support tracking register names in rules.

* Added win32k type generator definitions.

* Added a VMSS address space for vmware vmss files.

R=parki.san@gmail.com

Review URL: https://codereview.appspot.com/131930043

MANIFEST.in

@@ -1,8 +1,8 @@
 include AUTHORS.txt CREDITS.txt LEGAL.txt LICENSE.txt README.rst README.md
 exclude .gitignore
 exclude *.pyc
-exclude .git
-exclude build
+recursive-exclude .git
+recursive-exclude build
 
 #recursive-include contrib *
 recursive-include docs *

rekall/addrspace.py

@@ -156,7 +156,7 @@ def _get_address_ranges(self):
         contiguous_poffset = 0
         total_length = 0
         for (voffset, poffset, length) in self.get_available_addresses():
-            # This can take sometime as we enumerate all the address ranges.
+            # This can take some time as we enumerate all the address ranges.
             if self.session:
                 self.session.report_progress(
                     "%(name)s: Merging Address Ranges %(spinner)s",

rekall/plugins/addrspaces/intel.py

@@ -35,6 +35,17 @@
                      help="The DTB physical address.")
 
 
+class PhysicalAddress(object):
+    """A class to represent a physical address.
+
+    This object is supposed to be an opaque object which passes data between the
+    various query functions of an address space into the read function.
+
+    i.e. one can receive the PhysicalAddress() object by calling vtop() and then
+    pass it back to the read() method.
+    """
+
+
 class IA32PagedMemory(addrspace.PagedReader):
     """ Standard x86 32 bit non PAE address space.
 
@@ -114,7 +125,6 @@ def pde_index(self, vaddr):
     def get_pde(self, vaddr):
         '''
         Return the Page Directory Entry for the given virtual address.
-        If caching
 
         Bits 31:12 are from CR3
         Bits 11:2 are bits 31:22 of the linear address
@@ -152,7 +162,6 @@ def get_four_meg_paddr(self, vaddr, pde_value):
         '''
         return  (pde_value & 0xffc00000) | (vaddr & 0x3fffff)
 
-
     def vtop(self, vaddr):
         '''
         Translates virtual addresses into physical offsets.

rekall/plugins/addrspaces/vmem.py

@@ -63,6 +63,42 @@ def __init__(self, base=None, **kwargs):
             self.runs.insert((v * 0x1000, p * 0x1000, l * 0x1000))
 
 
+class VMSSAddressSpace(addrspace.RunBasedAddressSpace):
+    __image = True
+
+    def __init__(self, base=None, **kwargs):
+        self.as_assert(base != None, "No base address space provided")
+        super(VMSSAddressSpace, self).__init__(base=base, **kwargs)
+
+        vmss_profile = VMWareProfile(session=self.session)
+
+        self.header = vmss_profile._VMWARE_HEADER(vm=self.base)
+        self.as_assert(
+            self.header.Magic in [
+                0xbed2bed0, 0xbad1bad1, 0xbed2bed2, 0xbed3bed3],
+            "Invalid VMware signature: {0:#x}".format(self.header.Magic))
+
+        region_count = self.header.GetTags("memory", "regionsCount")
+
+        # Fill in the runs list from the header.
+        virtual_offsets = self.header.GetTags("memory", "regionPPN")
+        lengths = self.header.GetTags("memory", "regionSize")
+        mem_regions = self.header.GetTags("memory", "Memory")
+
+        # Single region case.
+        if region_count and region_count[0] == 0:
+            if not mem_regions:
+                raise IOError("Unable to locate mem region tag in VMSS file.")
+
+            self.runs.insert(
+                (0, mem_regions[0].obj_offset, mem_regions[0].length))
+        else:
+            for v, l, m in zip(
+                    virtual_offsets, lengths, mem_regions):
+                self.runs.insert(
+                    (v * 0x1000, m.obj_offset, l * 0x1000))
+
+
 class _VMWARE_HEADER(obj.Struct):
     """Add convenience methods to the header."""
 
@@ -200,7 +236,7 @@ def Initialize(cls, profile):
                 # by DataDiskSize.
                 'Data': [lambda x: x.Padding.obj_end, ["String", dict(
                     term=None,
-                    length=lambda x: x.DataDiskSize
+                    length=lambda x: x.DataDiskSize,
                     )]],
                 }],
 

rekall/plugins/common/address_resolver.py

@@ -53,6 +53,20 @@ def __init__(self, **kwargs):
         super(AddressResolverMixin, self).__init__(**kwargs)
         self.profiles = {}
 
+    def NormalizeModuleName(self, module):
+        try:
+            module_name = module.name
+        except AttributeError:
+            module_name = module
+
+        module_name = unicode(module_name)
+        module_name = re.split(r"[/\\]", module_name)[-1]
+
+        return module_name.lower()
+
+    def _EnsureInitialized(self):
+        """Initialize this address resolver."""
+
     def _ParseAddress(self, name):
         m = self.ADDRESS_NAME_REGEX.match(name)
         if m:
@@ -181,12 +195,17 @@ def format_address(self, address, max_distance=0x1000):
         Returns an empty string if the address is not in a containing module, or
         if the nearest known symbol is farther than max_distance away.
         """
+        _ = address
+        _ = max_distance
+        return ""
 
     def get_nearest_constant_by_address(self, address):
         """Searches for a known symbol at an address lower than this.
 
         Returns a tuple (nearest_offset, full_name of symbol).
         """
+        _ = address
+        return (0xFFFFFFFFFF, "")
 
     def search_symbol(self, pattern):
         """Searches symbols for the pattern.

rekall/plugins/core.py

@@ -147,7 +147,7 @@ def _split_into_paragraphs(self, string, dedent):
 
     def split_into_paragraphs(self, string, dedent=0, wrap=50):
         for paragraph, leading_space in self._split_into_paragraphs(
-            string, dedent):
+                string, dedent):
             paragraph = textwrap.wrap("\n".join(paragraph), wrap)
             yield "\n".join([(" " * leading_space + x) for x in paragraph])
 
@@ -217,7 +217,7 @@ def render_item_info(self, item, renderer):
     def _clean_up_doc(self, doc, dedent=0):
         clean_doc = []
         for paragraph in self.split_into_paragraphs(
-            " " * dedent + doc, dedent=dedent, wrap=70):
+                " " * dedent + doc, dedent=dedent, wrap=70):
             clean_doc.append(paragraph)
 
         return "\n".join(clean_doc)
@@ -765,7 +765,7 @@ def render(self, renderer):
         offset = 0
         resolver = self.session.address_resolver
         for offset, hexdata, translated_data in utils.Hexdump(
-            data, width=self.width):
+                data, width=self.width):
 
             # Add a symbol name for the start of each row.
             comment = resolver.format_address(
@@ -840,7 +840,7 @@ def render(self, renderer):
             data = self.address_space.read(offset, 4096)
             for idx in self._GenerateHits(data):
                 for dump_offset, hexdata, translated_data in utils.Hexdump(
-                    data[idx-20:idx+20], width=self.context):
+                        data[idx-20:idx+20], width=self.context):
                     comment = ""
                     nearest_offset, symbol = (
                         resolver.get_nearest_constant_by_address(offset + idx))

rekall/plugins/darwin/address_resolver.py

@@ -38,17 +38,6 @@ def __init__(self, **kwargs):
         # Delay initialization until we need it.
         self._initialized = False
 
-    def NormalizeModuleName(self, module):
-        try:
-            module_name = module.name
-        except AttributeError:
-            module_name = module
-
-        module_name = unicode(module_name)
-        module_name = re.split(r"[/\\]", module_name)[-1]
-
-        return module_name.lower()
-
     def _EnsureInitialized(self):
         if self._initialized:
             return

rekall/plugins/guess_profile.py

@@ -158,9 +158,16 @@ def ScanProfiles(self):
 
         # If the file is a PE file, we simply return the PE address space.
         if pe_profile._IMAGE_DOS_HEADER(vm=address_space).NTHeader:
-            self.session.kernel_address_space = pe_vtypes.PEFileAddressSpace(
+            pe_as = pe_vtypes.PEFileAddressSpace(
                 base=address_space, profile=pe_profile)
 
+            self.session.kernel_address_space = pe_as
+            self.session.SetParameter("default_image_base", pe_as.image_base)
+
+            machine_type = pe_as.nt_header.FileHeader.Machine
+            if machine_type == "IMAGE_FILE_MACHINE_AMD64":
+                pe_profile.set_metadata("arch", "ADM64")
+
             return pe_profile
 
         for hit in ProfileScanner(address_space=address_space,

rekall/plugins/linux/address_resolver.py

@@ -57,17 +57,6 @@ def __init__(self, **kwargs):
         # Delay initialization until we need it.
         self._initialized = False
 
-    def NormalizeModuleName(self, module):
-        try:
-            module_name = module.name
-        except AttributeError:
-            module_name = module
-
-        module_name = unicode(module_name)
-        module_name = re.split(r"[/\\]", module_name)[-1]
-
-        return module_name.lower()
-
     def _EnsureInitialized(self):
         if self._initialized:
             return
@@ -179,7 +168,7 @@ def format_address(self, address, max_distance=0x1000):
             # Ensure address falls within the current module.
             containing_module = self._FindContainingModule(address)
             if (containing_module and address < containing_module.end and
-                0 < difference < max_distance):
+                    0 < difference < max_distance):
                 return "%s + %#x" % (
                     name, address - offset)
 

rekall/plugins/overlays/basic.py

@@ -67,11 +67,8 @@ def __init__(self, length=1024, max_length=1024000, term="\x00", **kwargs):
             length = length(self.obj_parent)
 
         self.term = term
-        if length > max_length:
-            logging.warn("%s@%#x truncated", self.obj_name, self.obj_offset)
-            length = 0
-
         self.length = int(length)
+        self.max_length = max_length
 
     @property
     def obj_end(self):
@@ -81,8 +78,15 @@ def startswith(self, other):
         return self.v().startswith(other)
 
     def v(self, vm=None):
+        # Make sure to protect ourselves before reading too much at once.
+        length = self.length
+        if self.length > self.max_length:
+            logging.warn("%s@%#x truncated", self.obj_name, self.obj_offset)
+            length = 0
+
+        # TODO: Make this read in chunks to support very large reads.
         vm = vm or self.obj_vm
-        data = vm.read(self.obj_offset, self.length)
+        data = vm.read(self.obj_offset, length)
         if self.term is not None:
             left, sep, _ = data.partition(self.term)
             data = left + sep

rekall/plugins/overlays/windows/pe_vtypes.py

@@ -810,13 +810,22 @@ def __init__(self, address_space=None, image_base=0, filename=None,
         # Use the session to load the pe profile.
         self.profile = self.session.LoadProfile("pe")
 
-        if address_space is None:
+        # If neither filename or address_space were provided we just get the
+        # session default.
+        if filename is None and address_space is None:
             address_space = self.session.GetParameter("default_address_space")
+            if address_space == None:
+                raise IOError("Filename or address_space not specified.")
 
-        if filename is None and address_space is None:
-            raise IOError("Filename or address_space not specified.")
+            self.vm = address_space
+            self.image_base = image_base
 
         elif address_space:
+            # Resolve the correct address space. This allows the address space
+            # to be specified from the command line (e.g. "P")
+            load_as = self.session.plugins.load_as(session=self.session)
+            address_space = load_as.ResolveAddressSpace(address_space)
+
             self.vm = address_space
             self.image_base = image_base
             if self.image_base == None:
@@ -935,6 +944,9 @@ def VersionInformation(self):
             for string in version_info.Strings():
                 yield unicode(string.Key), unicode(string.Value)
 
+    def VersionInformationDict(self):
+        return dict(self.VersionInformation())
+
     def Sections(self):
         for section in self.nt_header.Sections:
 
@@ -1039,7 +1051,7 @@ def __init__(self, **kwargs):
     def __str__(self):
         return "<PEFileAddressSpace @ %#x >" % self.image_base
 
-    def read(self, addr, length):
+    def read_partial(self, addr, length):
         # Not a particularly efficient algorithm, but probably fast enough since
         # usually there are not too many sections.
         for virtual_address, run_length, physical_address in self.runs:
@@ -1051,3 +1063,19 @@ def read(self, addr, length):
 
         # Otherwise just null pad the results.
         return '\x00' * length
+
+    def read(self, addr, length):
+        addr, length = int(addr), int(length)
+
+        result = ""
+        while length > 0:
+            data = self.read_partial(addr, length)
+            if not data:
+                data = "\x00"
+
+            result += data
+            length -= len(data)
+            addr += len(data)
+
+        return result
+

rekall/plugins/overlays/windows/tcpip_vtypes.py

@@ -603,10 +603,10 @@ def Initialize(cls, profile):
             if version == "6.0":
                 profile.add_overlay(tcpip_vtypes_vista_64)
                 profile.add_overlay({
-                        '_TCP_ENDPOINT': [None, {
-                                'Owner' : [0x210, ['pointer', ['_EPROCESS']]],
-                                }],
-                        })
+                    '_TCP_ENDPOINT': [None, {
+                        'Owner' : [0x210, ['pointer', ['_EPROCESS']]],
+                        }],
+                    })
 
             # Windows 7
             elif version >= "6.1":

rekall/plugins/overlays/windows/windows.py

@@ -51,14 +51,29 @@ def get_constant(self, name, is_address=False):
 
         return base_constant
 
+    def add_constants(self, relative_to_image_base=True, **kwargs):
+        """Add new constants to this profile.
+
+        Args:
+
+          - relative_to_image_base: If True, the constants are specified
+            relative to the image base. Otherwise constants are absolute
+            addresses.
+        """
+        for k, v in kwargs.items():
+            if not relative_to_image_base:
+                kwargs[k] = (v) - self.GetImageBase()
+
+        super(RelativeOffsetMixin, self).add_constants(**kwargs)
+
     def get_nearest_constant_by_address(self, address, below=True):
         if address < self.GetImageBase():
             return 0, ""
 
         try:
             offset, name = super(
                 RelativeOffsetMixin, self).get_nearest_constant_by_address(
-                address - self.GetImageBase(), below=below)
+                    address - self.GetImageBase(), below=below)
 
             return offset + self.GetImageBase(), name
         except ValueError: