Updated the profile repository maintainance tool:

- It is now its own plugin - this way it can be built with the rest of Rekall.
- A new repository config file allows customization of profile manipulation through plugins.
- Added Profile Loaders to customize how various profile sections are handled.

New profile loaders introduced:
$MERGE - tells Rekall to merge in another profile into this profile at runtime.
$DYNAMIC_CONSTANTS - tells Rekall to load some constants as dynamic constants - if the constants are not defined in the profile, they will be automatically calculated using a sequence of detectors.

A new detector is DisassembleConstantMatcher which will extract the value of a constant using a disassembler template. Currently implemented using OSX _llinfo_arp as an example:
$DYNAMIC_CONSTANTS:
  _llinfo_arp:
    - type: DisassembleConstantMatcher
      args:
        start: __kernel__!_arp_init
        base: __kernel__
        rules: [
        # mov qword ptr [rip + 0x40e948], 0
        {'mnemonic': 'MOV', 'operands': [
           {'scale': 1, 'address': $out, 'base': 'RIP', 'type': 'MEM'},
           {'type': 'IMM', 'target': 0, 'size': 8}],
        }
        ]

- Added $DYNAMIC_STRUCT handler which can build structs based on disassembly patterns. Added several signatures for _IMAGE_IN_SESSION (win7, 8, 10).
- Fixed ipython inspector to report current doc strings and args from introspection - we no longer use the help profile.

R=parki.san@gmail.com

Review URL: https://codereview.appspot.com/279410043 .

.gitignore

@@ -14,6 +14,7 @@ dist/
 rekall.egg-info
 rekall_gui.egg-info
 rekall_core.egg-info
+*.egg-info
 
 # Xcode & OS X-related stuff
 .DS_Store

rekall-core/MANIFEST.in

@@ -11,3 +11,7 @@ include _version.py
 exclude .gitignore
 exclude *.pyc
 include rekall/_version.py
+
+# Drivers for OSX and windows.
+recursive-include resources *.sys
+recursive-include resources *.tgz

rekall-core/rekall/io_manager.py

@@ -122,7 +122,7 @@ def __init__(self, urn=None, mode="r", session=None, pretty_print=True,
     @property
     def inventory(self):
         if self._inventory is None:
-            self._inventory = self.GetData("inventory")
+            self._inventory = self.GetData("inventory", default={})
 
         return self._inventory
 
@@ -172,6 +172,12 @@ def Metadata(self, path):
         inventory = self.inventory.get("$INVENTORY", {})
         return inventory.get(path, {})
 
+    def SetMetadata(self, name, options):
+        existing_options = self.Metadata(name)
+        existing_options.update(options)
+        self.inventory.setdefault("$INVENTORY", {})[name] = existing_options
+        self.FlushInventory()
+
     def FlushInventory(self):
         """Write the inventory to the storage."""
         self.inventory.setdefault("$METADATA", dict(
@@ -322,6 +328,29 @@ def __init__(self, urn=None, **kwargs):
         self.check_dump_dir(self.dump_dir)
         self.canonical_name = os.path.basename(self.dump_dir)
 
+    @property
+    def inventory(self):
+        # In DirectoryIOManager the inventory reflects the directory structure.
+        if self._inventory is None:
+            self._inventory = self.GetData("inventory", default={})
+            if not self._inventory:
+                self._inventory = self.RebuildInventory()
+
+        return self._inventory
+
+    def RebuildInventory(self):
+        """Rebuild the inventory file."""
+        result = {
+            "$METADATA": dict(
+                Type="Inventory",
+                ProfileClass="Inventory"),
+            "$INVENTORY": {},
+        }
+        for member in self.ListFiles():
+            result["$INVENTORY"][member] = self.Metadata(member)
+
+        return result
+
     def CheckInventory(self, path):
         """Checks the validity of the inventory and if the path exists in it.
 
@@ -330,16 +359,19 @@ def CheckInventory(self, path):
         if a profile exists in this repository.
         """
         if self.ValidateInventory():
-            path = self._GetAbsolutePathName(path)
+            path = self.GetAbsolutePathName(path)
             return os.access(path, os.R_OK) or os.access(path + ".gz", os.R_OK)
         return False
 
     def Metadata(self, path):
-        path = self._GetAbsolutePathName(path)
+        path = self.GetAbsolutePathName(path)
         try:
             try:
                 st = os.stat(path + ".gz")
             except OSError:
+                if os.path.isdir(path):
+                    return {}
+
                 st = os.stat(path)
 
             return dict(LastModified=st.st_mtime)
@@ -356,7 +388,7 @@ def check_dump_dir(self, dump_dir=None):
         if not os.path.isdir(dump_dir):
             raise IOManagerError("%s is not a directory" % self.dump_dir)
 
-    def _GetAbsolutePathName(self, name):
+    def GetAbsolutePathName(self, name):
         path = os.path.normpath(
             os.path.join(self.dump_dir, self.version, name))
 
@@ -372,24 +404,28 @@ def EnsureDirectoryExists(self, dirname):
             pass
 
     def ListFiles(self):
-        for root, _, files in os.walk(self.dump_dir):
+        top_level = os.path.join(self.dump_dir, self.version)
+        for root, _, files in os.walk(top_level):
             for f in files:
                 path = os.path.normpath(os.path.join(root, f))
 
+                if path.endswith(".gz"):
+                    path = path[:-3]
+
                 # Return paths relative to the dump dir.
-                yield path[len(self.dump_dir) + 1:]
+                yield path[len(top_level) + 1:]
 
     def Create(self, name):
-        path = self._GetAbsolutePathName(name)
+        path = self.GetAbsolutePathName(name)
         self.EnsureDirectoryExists(os.path.dirname(path))
         return gzip.open(path + ".gz", "wb")
 
     def Destroy(self, name):
-        path = self._GetAbsolutePathName(name)
+        path = self.GetAbsolutePathName(name)
         return shutil.rmtree(path)
 
     def Open(self, name):
-        path = self._GetAbsolutePathName(name)
+        path = self.GetAbsolutePathName(name)
         try:
             result = open(path, "rb")
         except IOError:

rekall-core/rekall/io_manager_test.py

@@ -0,0 +1,80 @@
+# Rekall Memory Forensics
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Author: Michael Cohen scudette@google.com
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+import gzip
+import os
+
+from rekall import constants
+from rekall import io_manager
+from rekall import testlib
+
+
+class IOManagerTest(testlib.RekallBaseUnitTestCase):
+    """Test the IO Manager."""
+
+    DATA = {
+        "foo.gz": "hello",
+        "bar": "goodbye"
+    }
+
+    def setUp(self):
+        super(IOManagerTest, self).setUp()
+
+        # Create a new repository in the temp directory.
+        self.version = constants.PROFILE_REPOSITORY_VERSION
+        for filename, data in self.DATA.iteritems():
+            path = os.path.join(self.temp_directory, self.version,
+                                filename)
+
+            if path.endswith("gz"):
+                opener = gzip.open
+            else:
+                opener = open
+
+            try:
+                os.makedirs(os.path.dirname(path))
+            except (OSError, IOError):
+                pass
+
+            with opener(path, "wb") as fd:
+
+                fd.write(data)
+
+    def testDirectoryIOManager(self):
+        manager = io_manager.DirectoryIOManager(
+            self.temp_directory,
+            session=self.MakeUserSession())
+
+        # Cant decode from json.
+        self.assertEqual(manager.GetData("foo"), None)
+        self.assertEqual(manager.GetData("foo", raw=True),
+                         "hello")
+
+        # Test ListFiles().
+        self.assertListEqual(list(manager.ListFiles()),
+                             ["foo", "bar"])
+
+        # Storing a data structure.
+        data = dict(a=1)
+        manager.StoreData("baz", data)
+        self.assertDictEqual(manager.GetData("baz"),
+                             data)
+
+        self.assertTrue(
+            isinstance(manager.GetData("baz", raw=True), basestring))

rekall-core/rekall/ipython_support.py

@@ -18,7 +18,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #
 
-"""Support IPython 1.0."""
+"""Support IPython 3.0."""
 
 # pylint: disable=protected-access
 
@@ -28,6 +28,9 @@
 import readline
 
 from rekall import constants
+from rekall import config
+from rekall import session
+from IPython.core import oinspect
 from IPython.terminal import embed
 try:
     from traitlets.config.loader import Config
@@ -65,11 +68,12 @@ def RekallCompleter(self, text):
         if (global_matches and len(global_matches) == 1 and
                 len(command_parts) > 1):
 
-            # Get the object and ask it about the list of default args.
+            # Get the object and ask it about the list of args that it supports.
             obj = self.namespace.get(global_matches.pop())
             if obj:
                 try:
-                    matches = ["%s=" % x for x in obj.get_default_arguments()]
+                    matches = [
+                        "%s=" % x["name"] for x in obj.Metadata()["arguments"]]
                     return [x for x in matches if x.startswith(text)]
                 except Exception:
                     pass
@@ -82,12 +86,75 @@ def RekallCompleter(self, text):
         logging.debug(e)
 
 
+class RekallObjectInspector(oinspect.Inspector):
+    """Rekall specific object inspector.
+
+    Rekall populates the environment with "plugin runners" which are proxies of
+    the actual plugin that will be invoked. The exact plugin will be invoked
+    depending on the profile availability.
+
+    In order to make ipython's ? and ?? operators work, we need to implement
+    specialized inspection to present the doc strings and arg list of the actual
+    plugin.
+    """
+
+    pinfo_fields1 = oinspect.Inspector.pinfo_fields1 + [
+        ("Link", "link"),
+        ("Parameters", "parameters")]
+
+    def format_parameters(self, plugin_class):
+        displayfields = []
+        command_metadata = config.CommandMetadata(plugin_class).Metadata()
+        for arg in command_metadata["arguments"]:
+            desc = arg["help"]
+            try:
+                desc += " (type: %s)" % arg["type"]
+            except KeyError:
+                pass
+
+            displayfields.append(("  " + arg["name"], desc))
+
+        return self._format_fields(displayfields)
+
+    def plugin_info(self, runner, **kwargs):
+        """Generate info dict for a plugin from a plugin runner."""
+        plugin_class = getattr(
+            runner.session.plugins, runner.plugin_name)._target
+
+        result = oinspect.Inspector.info(self, plugin_class, **kwargs)
+        result["file"] = oinspect.find_file(plugin_class)
+        result["type_name"] = "Rekall Plugin (%s)" % plugin_class.__name__
+        result["parameters"] = self.format_parameters(plugin_class)
+        result["docstring"] = oinspect.getdoc(plugin_class)
+        result["link"] = (
+            "http://www.rekall-forensic.com/epydocs/%s.%s-class.html" % (
+                plugin_class.__module__, plugin_class.__name__))
+
+        # Hide these two fields.
+        result["init_definition"] = None
+        result["string_form"] = None
+
+        return result
+
+    def info(self, obj, **kwargs):
+        if isinstance(obj, session.PluginRunner):
+            # Delegate info generation for PluginRunners.
+            return self.plugin_info(obj, **kwargs)
+
+        result = oinspect.Inspector.info(self, obj, **kwargs)
+        result["link"] = result["parameters"] = None
+        return result
+
+
 class RekallShell(embed.InteractiveShellEmbed):
     display_banner = constants.BANNER
 
     def atexit_operations(self):
         self.user_global_ns.session.Flush()
 
+    def init_inspector(self):
+        self.inspector = RekallObjectInspector()
+
 
 def Shell(user_session):
     # This should bring back the old autocall behaviour. e.g.: