Initial commit of AFF4 imager and address space.

BUG=
R=adam.sindelar@gmail.com

Review URL: https://codereview.appspot.com/210740043

debian/source/format

@@ -0,0 +1 @@
+1.0

rekall/plugins/addrspaces/__init__.py

@@ -31,3 +31,8 @@
     import rekall.plugins.addrspaces.win32
 except ImportError:
     pass
+
+try:
+    import rekall.plugins.addrspaces.aff4
+except ImportError:
+    pass

rekall/plugins/addrspaces/aff4.py

@@ -0,0 +1,130 @@
+# Rekall Memory Forensics
+#
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Authors:
+# Copyright (C) 2015 Michael Cohen <scudette@google.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+
+"""This Address Space allows us to open aff4 images.
+
+AFF4 images are produced by the Rekall memory acquisition tools (Pmem and
+friends).
+
+For this address space to work:
+
+pip install pyaff4
+
+"""
+import logging
+
+from rekall import addrspace
+from rekall.plugins.addrspaces import standard
+
+from pyaff4 import data_store
+from pyaff4 import zip
+from pyaff4 import lexicon
+from pyaff4 import plugins
+
+
+# Control the logging level for the pyaff4 library logger.
+LOGGER = logging.getLogger("pyaff4")
+LOGGER.setLevel(logging.ERROR)
+
+
+class AFF4StreamWrapper(object):
+    def __init__(self, stream):
+        self.stream = stream
+
+    def read(self, offset, length):
+        self.stream.seek(offset)
+        return self.stream.read(length)
+
+    def end(self):
+        return self.stream.Size()
+
+
+class AFF4AddressSpace(addrspace.CachingAddressSpaceMixIn,
+                       addrspace.MultiRunBasedAddressSpace):
+    """Handle AFF4Map or AFF4Image type streams."""
+    __name = "aff4"
+    __image = True
+
+    order = standard.FileAddressSpace.order - 10
+
+    def __init__(self, filename=None, **kwargs):
+        super(AFF4AddressSpace, self).__init__(**kwargs)
+
+        self.as_assert(self.base == None,
+                       "Must stack on another address space")
+
+        path = filename or self.session.GetParameter("filename")
+        self.as_assert(path != None, "Filename must be specified")
+
+        self.image = None
+        self.phys_base = self
+        try:
+            self._LoadAFF4Volume(path)
+        except IOError:
+            raise addrspace.ASAssertionError(
+                "Unable to open AFF4 volume")
+
+    def _LoadAFF4Volume(self, path):
+        self.resolver = data_store.MemoryDataStore()
+        with zip.ZipFile.NewZipFile(self.resolver, path):
+            # We are searching for images with the physical memory category.
+            for (subject, _, value) in self.resolver.QueryPredicate(
+                    lexicon.AFF4_CATEGORY):
+                if value == lexicon.AFF4_MEMORY_PHYSICAL:
+                    self._LoadMemoryImage(subject)
+                    break
+
+        self.as_assert(self.image is not None,
+                       "No physical memory categories found.")
+
+        # Attempt to load any page files if there are any.
+        for (subject, _, value) in self.resolver.QueryPredicate(
+                lexicon.AFF4_CATEGORY):
+            if value == lexicon.AFF4_MEMORY_PAGEFILE:
+                pagefile_stream = self.resolver.AFF4FactoryOpen(subject)
+
+                self.pagefile_offset = self.end() + 0x10000
+                self.pagefile_end = (
+                    self.pagefile_offset + pagefile_stream.Size())
+
+                self.add_run(
+                    self.pagefile_offset, 0, pagefile_stream.Size(),
+                    AFF4StreamWrapper(pagefile_stream))
+
+                logging.info(
+                    "Added %s as pagefile", subject)
+
+    def _LoadMemoryImage(self, image_urn):
+        aff4_stream = self.resolver.AFF4FactoryOpen(image_urn)
+        self.image = AFF4StreamWrapper(aff4_stream)
+
+        # Add the ranges if this is a map.
+        try:
+            for map_range in aff4_stream.GetRanges():
+                self.runs.insert((map_range.map_offset,
+                                  map_range.map_offset,
+                                  map_range.length,
+                                  self.image))
+        except AttributeError:
+            self.runs.insert((0, 0, aff4_stream.Size(), self.image))
+
+        logging.info("Added %s as physical memory", image_urn)

rekall/plugins/addrspaces/amd64.py

@@ -28,6 +28,7 @@
 import logging
 import struct
 
+from rekall import addrspace
 from rekall import config
 from rekall import obj
 from rekall.plugins.addrspaces import intel
@@ -294,7 +295,10 @@ def __init__(self, ept=None, **kwargs):
         # A dummy DTB is passed to the base class so the DTB checks on
         # IA32PagedMemory don't bail out. We require the DTB to never be used
         # for page translation outside of get_pml4e.
-        super(VTxPagedMemory, self).__init__(dtb=0xFFFFFFFF, **kwargs)
+        try:
+            super(VTxPagedMemory, self).__init__(dtb=0xFFFFFFFF, **kwargs)
+        except TypeError:
+            raise addrspace.ASAssertionError()
 
         # Reset the DTB, in case a plugin or AS relies on us providing one.
         self.dtb = None

rekall/plugins/addrspaces/pmem.py

@@ -331,7 +331,8 @@ def __init__(self, base=None, filename=None, **kwargs):
                     self.runs.insert((offset, offset, pages * 0x1000))
         except IOError:
             # Apparently we're not dealing with Pmem.
-            raise TypeError("File at %s is not a pmem device." % path)
+            raise addrspace.ASAssertionError(
+                "File at %s is not a pmem device." % path)
 
     def write(self, *_, **__):
         raise NotImplementedError("Writes to Pmem aren't supported yet.")

rekall/plugins/core.py

@@ -474,8 +474,7 @@ def GuessAddressSpace(self, base_as=None, **kwargs):
                     found = True
                     break
                 except (AssertionError,
-                        addrspace.ASAssertionError,
-                        TypeError), e:
+                        addrspace.ASAssertionError), e:
                     logging.debug("Failed instantiating %s: %s",
                                   cls.__name__, e)
                     error.append_reason(cls.__name__, e)

rekall/plugins/linux/common.py

@@ -84,7 +84,7 @@ def vm_kernel_slide_hits(self):
 
         for hit in SlideScanner(
                 address_space=self.physical_address_space,
-                session=self.session).scan():
+                session=self.session).scan(maxlen=100*1024*1024):
             vm_kernel_slide = int(hit - expected_physical_offset)
 
             yield vm_kernel_slide

rekall/plugins/overlays/basic.py

@@ -675,8 +675,6 @@ def __init__(self, mode=None, args=None, **kwargs):
         else:
             self.distorm_mode = None
 
-        self.decompose_cache = []
-
     def __int__(self):
         return self.obj_offset
 
@@ -767,14 +765,6 @@ def Decompose(self, instructions=10, size=None):
           size: Stop after decoding this much data. If specified we ignore
             the instructions parameter.
         """
-        if self.distorm_mode:
-            if len(self.decompose_cache) < instructions:
-                self.decompose_cache = list(self._Decompose(
-                    instructions=instructions, size=size))
-
-        return self.decompose_cache
-
-    def _Decompose(self, instructions=10, size=None):
         overlap = 0x100
         data = ''
         offset = self.obj_offset

rekall/plugins/tools/profile_tool.py

@@ -397,7 +397,8 @@ def render(self, renderer):
                            profile_class=self.profile_class).Convert()
 
             try:
-                input = io_manager.Factory(self.source, mode="r")
+                input = io_manager.Factory(self.source, session=self.session,
+                                           mode="r")
             except IOError:
                 logging.critical("Input profile file %s could not be opened.",
                                  self.source)

rekall/plugins/windows/registry/registry.py

@@ -302,17 +302,17 @@ def open_subkey(self, subkey_name):
             if unicode(subkey.Name).lower() == subkey_name.lower():
                 return subkey
 
-        return obj.NoneObject("Couldn't find subkey {0} of {1}".format(
-            subkey_name, self.Name))
+        return obj.NoneObject("Couldn't find subkey {0} of {1}",
+                              subkey_name, self.Name)
 
     def open_value(self, value_name):
         """Opens our direct child."""
         for value in self.values():
             if value.Name == value_name:
                 return value
 
-        return obj.NoneObject("Couldn't find subkey {0} of {1}".format(
-            value_name, self.Name))
+        return obj.NoneObject("Couldn't find subkey {0} of {1}",
+                              value_name, self.Name)
 
     def subkeys(self):
         """Enumeate all subkeys of this key.

rekall/session.py

@@ -921,7 +921,8 @@ def __init__(self, env=None, use_config_file=True, session_name=None,
             session=self,
 
             # Prepopulate the namespace with our most important modules.
-            profile=self.profile, v=self.v,
+            profile=self.profile,
+            v=self.v,
 
             # Some useful modules which should be available always.
             sys=sys, os=os,

rekall/testlib.py

@@ -166,7 +166,9 @@ def LaunchExecutable(self, config_options, retries=2):
                 output = open(tmp_filename).read(10 * 1024 * 1024)
                 output = output.decode("utf8", "ignore")
 
-                if output == "" and retries > 0:
+                # Travis often kills a test for lack of resources - we just
+                # retry it a couple of times.
+                if output in ["", "Killed"] and retries > 0:
                     return self.LaunchExecutable(
                         config_options, retries=retries-1)
 

rekall/utils.py

@@ -196,6 +196,7 @@ def __init__(self, max_size=10, kill_cb=None, lock=False):
         self.lock = None
         if lock:
             self.lock = threading.RLock()
+        self.hits = self.misses = 0
 
     @Synchronized
     def Expire(self):
@@ -266,8 +267,12 @@ def Get(self, key):
             node, item = self._hash[key]
             self._age.Unlink(node)
             self._age.AppendNode(node)
+            self.hits += 1
         except ValueError:
             raise KeyError(key)
+        except KeyError:
+            self.misses += 1
+            raise
 
         return item
 

tools/pmem/.gitignore

@@ -0,0 +1,47 @@
+# Compiled Dynamic libraries
+# Compiled Object files
+# Compiled Static libraries
+# Executables
+# Fortran module files
+# Precompiled Headers
+# autoconf related files.
+*.a
+*.app
+*.dll
+*.dylib
+*.exe
+*.gch
+*.la
+*.lai
+*.lib
+*.lo
+*.mod
+*.o
+*.obj
+*.out
+*.pch
+*.slo
+*.so
+*.zip
+*_test
+*~
+.depend
+/autom4te.cache
+doxygen/
+pmem_imager
+test_filename.bin
+
+# autoconf related files.
+Makefile
+config.log
+config.status
+.deps
+.libs
+autom4te.cache
+config.h
+stamp-h1
+libtool
+
+# Built targets
+linpmem
+winpmem.exe

tools/pmem/LICENSE

@@ -0,0 +1,16 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+Note that the Apache license applies only to the Pmem memory acquisition
+tools. The rest of Rekall is still only available under the GPL.