Pre-release fixes.

* Added a file() efilter function.
* Updated version script to control Debian package version.
* Added osquery to the linux deb build. Run it off the system otherwise.

Review URL: https://codereview.appspot.com/322480043 .

debian/changelog

@@ -3,19 +3,4 @@ rekall-forensic (1.7.0) RELEASED; urgency=low
   [ Rekall Team ]
   * Release 1.7.0 Hurricane Ridge
 
- -- Rekall Team <rekall-dev@googlegroups.com>  Fri, 4 August 2017 8:46:37 +0000
-
-
-rekall-forensic (1.6.0) RELEASED; urgency=low
-
-  [ Rekall Team ]
-  * Release 1.6.0 Gotthard
-
- -- Rekall Team <rekall-dev@googlegroups.com>  Fri, 4 November 2016 8:46:37 +0000
-
-rekall-forensic (1.5.3) RELEASED; urgency=low
-
-  [ Rekall Team ]
-  * Release 1.5.3 Etzel
-
- -- Rekall Team <rekall-dev@googlegroups.com>  Wed, 10 August 2016 8:46:37 +0000
+ -- Rekall Team <rekall-dev@googlegroups.com>  Mon, 7 Aug 2017 3:38:43 -0000

debian/changelog.in

@@ -0,0 +1,6 @@
+rekall-forensic (%(version)s) RELEASED; urgency=low
+
+  [ Rekall Team ]
+  * Release %(version)s %(codename)s
+
+ -- Rekall Team <rekall-dev@googlegroups.com>  %(debian_ts)s

debian/rules

@@ -8,3 +8,8 @@ override_dh_strip:
 
 override_dh_virtualenv:
 	dh_virtualenv --python python2.7 --preinstall 'setuptools>36' --preinstall 'pip>=9.0' --preinstall 'wheel'
+
+
+override_dh_prep:
+	echo "Copy osquery into the resources tree"
+	cp /usr/bin/osqueryi rekall-core/resources

rekall-agent/rekall_agent/agent.py

@@ -569,20 +569,21 @@ def _Notify(self, event):
                 pass
 
 
-class AgentInfo(common.AbstractAgentCommand):
+class SystemInfo(plugin.TypedProfileCommand, plugin.Command):
     """Just emit information about the agent.
 
     The output format is essentially key value pairs. This is useful for efilter
     queries.
     """
-    name = "agent_info"
+    name = "system_info"
+    mode = "mode_live"
 
     table_header = [
-        dict(name="key"),
+        dict(name="key", width=20),
         dict(name="value")
     ]
 
     def collect(self):
         uname = UnameImpl.from_current_system(session=self.session)
-        for k, v in uname.to_primitive().iteritems():
+        for k, v in uname.to_primitive(with_type=False).iteritems():
             yield dict(key=k, value=v)

rekall-core/rekall/plugins/common/efilter_plugins/ipython.py

@@ -40,6 +40,11 @@ def pager(self, line, cell=None):
         if " " in line:
             _, line_end = line.split(" ", 1)
         else:
+            # A bare pager magic with pager already set, means to clear it.
+            if session.GetParameter("pager"):
+                session.SetParameter("pager", None)
+                return
+
             line_end = "less"
 
         session.SetParameter("pager", line_end)

rekall-core/rekall/plugins/common/efilter_plugins/search.py

@@ -87,13 +87,13 @@
 
 from efilter.protocols import applicative
 from efilter.protocols import associative
-from efilter.protocols import counted
 from efilter.protocols import repeated
 from efilter.protocols import structured
 
 from rekall import obj
 from rekall import plugin
 from rekall import testlib
+from rekall.plugins.response import common
 from rekall.plugins.overlays import basic
 from rekall.plugins.common.efilter_plugins import helpers
 from rekall.ui import identity as identity_renderer
@@ -443,6 +443,14 @@ def _get_scopes(self):
         scopes["timestamp"] = api.user_func(
             lambda x, **_: basic.UnixTimeStamp(value=x, session=self.session),
             arg_types=[float, int, long])
+
+        # This function is used to indicate that the string represents
+        # a filename. This will cause the agent to upload it if the
+        # user requested uploading files.
+        # > select file(path.filename.name).filename.name from glob("/*")
+        scopes["file"] = api.user_func(
+            lambda x: common.FileInformation(session=self.session, filename=x),
+            arg_types=[unicode, str])
         return scopes
 
     # IStructured implementation for EFILTER:

rekall-core/rekall/plugins/overlays/basic.py

@@ -86,8 +86,12 @@ def v(self, vm=None):
         vm = vm or self.obj_vm
         data = vm.read(self.obj_offset, length)
         if self.term is not None:
-            left, sep, _ = data.partition(self.term)
-            data = left + sep
+            try:
+                left, sep, _ = data.partition(self.term)
+                data = left + sep
+                # We can not split it, just return the full length.
+            except ValueError:
+                pass
 
         return data
 

rekall-core/rekall/plugins/response/common.py

@@ -89,7 +89,8 @@ def __init__(self, filename, filesystem=u"API", path_sep=None):
             self.path_sep = path_sep or self.default_path_sep
 
         else:
-            raise TypeError("Filename must be a string or file spec.")
+            raise TypeError("Filename must be a string or file spec not %s." % type(
+                filename))
 
     @property
     def dirname(self):

rekall-core/rekall/plugins/response/files.py

@@ -69,7 +69,7 @@ class IRStat(common.AbstractIRCommandPlugin):
 
     __args = [
         dict(name="paths", positional=True, type="Array",
-             help="Paths to hash."),
+             help="Paths to stat."),
     ]
 
     table_header = [
@@ -123,8 +123,10 @@ def calculate_hashes(self, hashes, file_info):
             for hasher in hashers.values():
                 hasher.update(data)
 
-        return [Hash(type=name, value=hasher.digest())
-                for name, hasher in hashers.iteritems()]
+        for key in list(hashers):
+            hashers[key] = hashers[key].hexdigest()
+
+        return hashers
 
     def collect(self):
         for path in self.plugin_args.paths:

rekall-core/rekall/plugins/response/osquery.py

@@ -35,6 +35,9 @@
 import platform
 import subprocess
 
+from distutils import spawn
+
+from rekall import plugin
 from rekall import resources
 from rekall.plugins.response import common
 
@@ -70,6 +73,10 @@ def try_to_find_osquery(self):
                 if os.access(result, os.R_OK):
                     return result
 
+            else:
+                # Try to find it somewhere on the system.
+                return spawn.find_executable("osqueryi")
+
             raise e
 
     def render(self, renderer):
@@ -79,6 +86,9 @@ def render(self, renderer):
         if osquery_path == None:
             osquery_path = self.try_to_find_osquery()
 
+        if not self.plugin_args.query:
+            raise plugin.PluginError("Query must be provided")
+
         self.session.logging.debug("Found OSQuery at %s" % osquery_path)
         self.json_result = json.loads(
             subprocess.check_output(

rekall-core/resources/rekall-agent-windows.bat

@@ -1,3 +1,5 @@
+echo off
+
 rem This batch script installs Rekall Agent as a windows service using
 rem the nssm tool which was originally downloaded from http://nssm.cc/
 

rekall-core/resources/rekall-agent.yaml

@@ -7,7 +7,7 @@ client:
   # Add intrinsic labels to this client here.
   labels:
     - All
-    - Linux
+    - Windows
 
   # Set to false to prevent the agent from polling - it will just
   # exit after a single poll. Note that the nanny will wake it anyway
@@ -23,4 +23,4 @@ client:
     base: http://127.0.0.1:8080/api/control
     path_prefix: /manifest
   # Persistent path to keep the same client.
-  writeback_path: rekall-agent-writeback.json
+  writeback_path: rekall-agent-writeback.json

rekall-core/setup.py

@@ -56,7 +56,7 @@ def find_data_files(source):
     "PyYAML == 3.11",
     "acora == 2.0",
     "arrow == 0.7.0",
-    "efilter == 1!1.5.2",
+    "rekall-efilter == 1.5.2",
     "intervaltree == 2.1.0",
     "pycrypto == 2.6.1",
     "pyelftools == 0.24",

version.py

@@ -5,7 +5,7 @@
 This program is used to manage versions. Prior to each release, please run it
 with update.
 """
-
+import arrow
 import argparse
 import json
 import os
@@ -138,14 +138,24 @@ def get_versions(version_file="version.yaml"):
 def escape_string(instr):
     return instr.replace('"""', r'\"\"\"')
 
+TEMPLATES = [
+    "debian/changelog.in"
+]
+
+
+def update_templates(version_data):
+    version_data["debian_ts"] = arrow.utcnow().format(
+        'ddd, D MMM YYYY h:mm:ss Z')
+    for path in TEMPLATES:
+        if not path.endswith(".in"):
+            continue
+
+        target = path[:-3]
+        with open(target, "wb") as outfd:
+            outfd.write(open(path).read() % version_data)
 
-def update(args):
-    if (args.version is None and
-            args.post is None and
-            args.rc is None and
-            args.codename is None):
-        raise AttributeError("You must set something in this release.")
 
+def update_version_files(args):
     data, version_path = get_config_file(args.version_file)
     version_data = data["version_data"]
     if args.version:
@@ -180,6 +190,18 @@ def update(args):
         with open(version_path, "wb") as fd:
             fd.write(contents)
 
+    update_templates(version_data)
+
+
+def update(args):
+    if (args.version is None and
+            args.post is None and
+            args.rc is None and
+            args.codename is None):
+        raise AttributeError("You must set something in this release.")
+
+    update_version_files(args)
+
 
 def main():
     parser = argparse.ArgumentParser()