Rewrote the apihooks plugin to be more robust and work on 64 bits.

BUG=

Review URL: https://codereview.appspot.com/104460043

.gitignore

@@ -5,6 +5,7 @@
 # Generic auto-generated build files
 *.pyc
 *.pyo
+*.o
 
 # Specific auto-generated build files
 /py27

rekall/addrspace.py

@@ -292,6 +292,8 @@ def __init__(self, **kwargs):
         self._cache = utils.FastStore(self.CACHE_SIZE)
 
     def read(self, addr, length):
+        addr, length = int(addr), int(length)
+
         result = ""
         while length > 0:
             data = self.read_partial(addr, length)

rekall/kb.py

@@ -144,15 +144,15 @@ def __init__(self, session):
         self.context = None
         self.Reset()
 
-    def _NormalizeModuleName(self, module):
+    def NormalizeModuleName(self, module):
         try:
             module_name = module.name
         except AttributeError:
             module_name = module
 
-        module_name = module_name.split("/")[-1]
-        module_name = module_name.split("\\")[-1]
-        result = unicode(module_name).split(".")[0]
+        module_name = unicode(module_name)
+        module_name = re.split(r"[/\\]", module_name)[-1]
+        result = module_name.split(".")[0]
         if result == "ntoskrnl":
             result = "nt"
 
@@ -167,7 +167,7 @@ def _ParseAddress(self, name):
                 if not module:
                     raise TypeError("Module name not specified.")
 
-                capture["module"] = self._NormalizeModuleName(module)
+                capture["module"] = self.NormalizeModuleName(module)
 
             if capture["op"] and not (capture["symbol"] or
                                       capture["address"] or
@@ -193,6 +193,32 @@ def _FindProcessVad(self, address):
         if task and self.vad:
             return self.vad.find_file_in_task(address, task)
 
+    def FindContainingModule(self, address):
+        """Finds the name of the module containing the specified address.
+
+        Returns:
+          A tuple of start address, end address, name
+          """
+        self._EnsureInitialized()
+
+        # The address may be in kernel space or user space.
+        containing_module = self._FindContainingModule(address)
+
+        if containing_module:
+            name = self.NormalizeModuleName(containing_module.name)
+            return containing_module.base, containing_module.size, name
+
+        # Maybe the address is in userspace.
+        containing_VAD = self._FindProcessVad(address)
+
+        # We find the vad and it
+        if containing_VAD:
+            start, end, name = containing_VAD
+            return start, end, self.NormalizeModuleName(name)
+
+        # If we dont know anything about the address just return Nones.
+        return None, None, None
+
     def GetState(self):
         return dict(modules=self.modules,
                     modules_by_name=self.modules_by_name,
@@ -241,7 +267,7 @@ def _EnsureInitialized(self):
             try:
                 self.modules = self.session.plugins.modules()
                 for module in self.modules.lsmod():
-                    module_name = self._NormalizeModuleName(module)
+                    module_name = self.NormalizeModuleName(module)
                     self.modules_by_name[module_name] = module
 
                     # Update the image base of our profiles.
@@ -263,7 +289,7 @@ def _EnsureInitialized(self):
     def _LoadProfile(self, module_name, profile):
         self._EnsureInitialized()
         try:
-            module_name = self._NormalizeModuleName(module_name)
+            module_name = self.NormalizeModuleName(module_name)
             # Try to get the profile directly from the local cache.
             if module_name in self.profiles:
                 return self.profiles[module_name]
@@ -320,7 +346,7 @@ def LoadProfileForModule(self, module):
         result = None
         module_base = module.base
 
-        module_name = self._NormalizeModuleName(module)
+        module_name = self.NormalizeModuleName(module)
         if module_name in self.profiles:
             return self.profiles[module_name]
 
@@ -483,7 +509,7 @@ def get_constant_by_address(self, address):
         address = obj.Pointer.integer_to_address(address)
         containing_module = self._FindContainingModule(address)
         if containing_module:
-            module_name = self._NormalizeModuleName(containing_module)
+            module_name = self.NormalizeModuleName(containing_module)
             module_profile = self.LoadProfileForName(module_name)
 
             if module_profile:
@@ -511,7 +537,7 @@ def get_nearest_constant_by_address(self, address):
         containing_module = self._FindContainingModule(address)
         if containing_module:
             nearest_offset = containing_module.base
-            full_name = module_name = self._NormalizeModuleName(
+            full_name = module_name = self.NormalizeModuleName(
                 containing_module)
 
             # Try to load the module profile.
@@ -531,7 +557,7 @@ def get_nearest_constant_by_address(self, address):
             vad_desc = self._FindProcessVad(address)
             if vad_desc:
                 start, _, full_name = vad_desc
-                module_name = self._NormalizeModuleName(full_name)
+                module_name = self.NormalizeModuleName(full_name)
                 nearest_offset = start
                 profile = self.LoadProfileForDll(start, full_name)
 

rekall/obj.py

@@ -88,7 +88,7 @@ def _DumpData(self):
         try:
             with utils.FileLock(open(self.filename, "rb")) as fd:
                 self._MergeData(json.loads(
-                        fd.read(), object_hook=self.JSONEncoder.as_set))
+                    fd.read(), object_hook=self.JSONEncoder.as_set))
 
                 with open(self.filename, "wb") as fd:
                     fd.write(json.dumps(self.data, cls=self.JSONEncoder))
@@ -323,7 +323,7 @@ def __init__(self, type_name=None, offset=0, vm=None, profile=None,
         """
         if kwargs:
             logging.error("Unknown keyword args {0} for {1}".format(
-                    kwargs, self.__class__.__name__))
+                kwargs, self.__class__.__name__))
 
         self.obj_type = type_name
 
@@ -337,6 +337,10 @@ def __init__(self, type_name=None, offset=0, vm=None, profile=None,
         self.obj_context = context or {}
         self.obj_session = session
 
+        if profile is None:
+            logging.critical("Profile must be provided to %s" % self)
+            raise RuntimeError("Profile must be provided")
+
     def __getstate__(self):
         # This method should generally return all the parameters passed to the
         # constructor. The type parameter describes the full MRO for this
@@ -428,7 +432,7 @@ def deref(self, vm=None):
     def dereference(self, vm=None):
         _ = vm
         return NoneObject("Can't dereference {0}".format(
-                self.obj_name), self.obj_profile)
+            self.obj_name), self.obj_profile)
 
     def reference(self):
         """Produces a pointer to this object.
@@ -453,7 +457,7 @@ def v(self, vm=None):
         """
         _ = vm
         return NoneObject("No value for {0}".format(
-                self.obj_name), self.obj_profile)
+            self.obj_name), self.obj_profile)
 
     def __str__(self):
         return utils.SmartStr(self)
@@ -571,7 +575,7 @@ def v(self, vm=None):
         data = self.obj_vm.read(self.obj_offset, self.size())
         if not data:
             return NoneObject("Unable to read {0} bytes from {1}".format(
-                    self.size(), self.obj_offset))
+                self.size(), self.obj_offset))
 
         (val,) = struct.unpack(self.format_string, data)
 
@@ -1227,7 +1231,7 @@ def m(self, attr):
             offset, cls = element
         else:
             return NoneObject(u"Struct {0} has no member {1}".format(
-                    self.obj_name, attr))
+                self.obj_name, attr))
 
         if callable(offset):
             ## If offset is specified as a callable its an absolute
@@ -2183,6 +2187,17 @@ def __getstate__(self):
             name=self.name
             )
 
+class TestProfile(Profile):
+    def _SetupProfileFromData(self, data):
+        """Let the test manipulate the data json object directly."""
+        self.data = data
+
+    def copy(self):
+        result = super(TestProfile, self).copy()
+        result.data = self.data
+
+        return result
+
 
 PROFILE_CACHE = utils.FastStore()
 

rekall/plugins/addrspaces/__init__.py

@@ -15,8 +15,15 @@
 from rekall.plugins.addrspaces import standard
 from rekall.plugins.addrspaces import vboxelf
 
-utils.ConditionalImport("rekall.plugins.addrspaces.accelerated")
-utils.ConditionalImport("rekall.plugins.addrspaces.ewf")
+try:
+    import rekall.plugins.addrspaces.accelerated
+except ImportError:
+    pass
+
+try:
+    import rekall.plugins.addrspaces.ewf
+except ImportError:
+    pass
 
 # If we are running on windows, load the windows specific AS.
 try:

rekall/plugins/addrspaces/ewf.py

@@ -127,13 +127,25 @@ class EWFAddressSpace(addrspace.CachingAddressSpaceMixIn,
     __image = True
 
     def __init__(self, base=None, filename=None, session=None, **kwargs):
-        self.as_assert(base != None, "No base address space provided")
+        if filename is None:
+            self.as_assert(base != None, "No base address space provided")
 
-        self.as_assert(base.read(0, 6) == "\x45\x56\x46\x09\x0D\x0A",
-                       "EWF signature not present")
+            self.as_assert(base.read(0, 6) == "\x45\x56\x46\x09\x0D\x0A",
+                           "EWF signature not present")
 
-        path = session.GetParameter("filename") or filename
-        fhandle = ewf_open([path])
+            filename = base.fname
+
+        self.filename = filename
+        fhandle = ewf_open([self.filename])
 
         super(EWFAddressSpace, self).__init__(
             fhandle=fhandle, session=session, **kwargs)
+
+    def __getstate__(self):
+        state = super(EWFAddressSpace, self).__getstate__()
+        state["filename"] = self.filename
+
+        return state
+
+    def __setstate__(self, state):
+        self.__init__(session=self.session, filename=state["filename"])

rekall/plugins/addrspaces/vboxelf.py

@@ -82,8 +82,8 @@ def __init__(self, **kwargs):
                 to_read = max(1000000, int(section.p_filesz))
                 data = self.base.read(section.p_offset, to_read)
                 data = data.split("\x00")[0]
-
-                self.LoadMetadata(data)
+                if data:
+                    self.LoadMetadata(data)
 
     def check_file(self):
         """Checks the base file handle for sanity."""
@@ -99,7 +99,7 @@ def LoadMetadata(self, data):
         """Load the WinPmem metadata from the elf file."""
         try:
             self.metadata.update(yaml.safe_load(data))
-        except yaml.YAMLError as e:
+        except (yaml.YAMLError, TypeError) as e:
             logging.error("Invalid file metadata, skipping: %s" % e)
             return
 

rekall/plugins/core.py

@@ -38,7 +38,6 @@
 from rekall import testlib
 from rekall import utils
 
-
 class DummyParser(object):
     """A dummy object used to collect all defined args."""
 

rekall/plugins/overlays/basic.py

@@ -74,7 +74,7 @@ def __init__(self, length=1024, max_length=1024000, term="\x00", **kwargs):
         self.length = int(length)
 
     def startswith(self, other):
-        return str(self).startswith(other)
+        return self.v().startswith(other)
 
     def v(self, vm=None):
         vm = vm or self.obj_vm
@@ -648,16 +648,23 @@ def __init__(self, mode=None, args=None, **kwargs):
         else:
             raise RuntimeError("Invalid mode %s" % self.mode)
 
+        self.decompose_cache = []
+
     def __int__(self):
         return self.obj_offset
 
     def __hash__(self):
         return self.obj_offset + hash(self.obj_vm)
 
     def __unicode__(self):
+        if self.mode == "AMD64":
+            format_string = "%0#14x  %s"
+        else:
+            format_string = "%0#10x  %s"
+
         result = []
-        for data in self.Disassemble():
-            result.append("0x%08X %20s %s" % data)
+        for offset, _, instruction in self.Disassemble():
+            result.append(format_string % (offset, instruction))
 
         return "\n".join(result)
 
@@ -730,19 +737,21 @@ def Decompose(self, instructions=10, size=None):
           size: Stop after decoding this much data. If specified we ignore
             the instructions parameter.
         """
-        overlap = 0x1000
+        if len(self.decompose_cache) < instructions:
+            self.decompose_cache = list(self._Decompose(
+                instructions=instructions, size=size))
+
+        return self.decompose_cache
+
+    def _Decompose(self, instructions=10, size=None):
+        overlap = 0x100
         data = ''
         offset = self.obj_offset
         count = 0
 
         while 1:
             data = self.obj_vm.read(offset, overlap)
 
-            # This could happen if we hit an unmapped page - we just
-            # abort.
-            if not data:
-                return
-
             op = obj.NoneObject()
             for op in distorm3.Decompose(offset, data, self.distorm_mode):
                 if op.address - offset > len(data) - 40: