From 43021cebdc3716819e36f86726931b063c487d3f Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Mon, 7 Aug 2017 12:52:26 -0700 Subject: [PATCH] Pre-release fixes. * Added a file() efilter function. * Updated version script to control Debian package version. * Added osquery to the linux deb build. Run it off the system otherwise. Review URL: https://codereview.appspot.com/322480043 . --- debian/changelog | 17 +-------- debian/changelog.in | 6 ++++ debian/rules | 5 +++ rekall-agent/rekall_agent/agent.py | 9 ++--- .../plugins/common/efilter_plugins/ipython.py | 5 +++ .../plugins/common/efilter_plugins/search.py | 10 +++++- rekall-core/rekall/plugins/overlays/basic.py | 8 +++-- rekall-core/rekall/plugins/response/common.py | 3 +- rekall-core/rekall/plugins/response/files.py | 8 +++-- .../rekall/plugins/response/osquery.py | 10 ++++++ .../resources/rekall-agent-windows.bat | 2 ++ rekall-core/resources/rekall-agent.yaml | 4 +-- rekall-core/setup.py | 2 +- version.py | 36 +++++++++++++++---- 14 files changed, 88 insertions(+), 37 deletions(-) create mode 100644 debian/changelog.in diff --git a/debian/changelog b/debian/changelog index 1155638f1..1a050023f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,19 +3,4 @@ rekall-forensic (1.7.0) RELEASED; urgency=low [ Rekall Team ] * Release 1.7.0 Hurricane Ridge - -- Rekall Team Fri, 4 August 2017 8:46:37 +0000 - - -rekall-forensic (1.6.0) RELEASED; urgency=low - - [ Rekall Team ] - * Release 1.6.0 Gotthard - - -- Rekall Team Fri, 4 November 2016 8:46:37 +0000 - -rekall-forensic (1.5.3) RELEASED; urgency=low - - [ Rekall Team ] - * Release 1.5.3 Etzel - - -- Rekall Team Wed, 10 August 2016 8:46:37 +0000 + -- Rekall Team Mon, 7 Aug 2017 3:38:43 -0000 diff --git a/debian/changelog.in b/debian/changelog.in new file mode 100644 index 000000000..70df92ee6 --- /dev/null +++ b/debian/changelog.in @@ -0,0 +1,6 @@ +rekall-forensic (%(version)s) RELEASED; urgency=low + + [ Rekall Team ] + * Release %(version)s %(codename)s + + -- Rekall Team %(debian_ts)s diff --git a/debian/rules b/debian/rules index a84a3748f..ea5fd0652 100755 --- a/debian/rules +++ b/debian/rules @@ -8,3 +8,8 @@ override_dh_strip: override_dh_virtualenv: dh_virtualenv --python python2.7 --preinstall 'setuptools>36' --preinstall 'pip>=9.0' --preinstall 'wheel' + + +override_dh_prep: + echo "Copy osquery into the resources tree" + cp /usr/bin/osqueryi rekall-core/resources diff --git a/rekall-agent/rekall_agent/agent.py b/rekall-agent/rekall_agent/agent.py index 57a87fba9..9d582c8eb 100644 --- a/rekall-agent/rekall_agent/agent.py +++ b/rekall-agent/rekall_agent/agent.py @@ -569,20 +569,21 @@ def _Notify(self, event): pass -class AgentInfo(common.AbstractAgentCommand): +class SystemInfo(plugin.TypedProfileCommand, plugin.Command): """Just emit information about the agent. The output format is essentially key value pairs. This is useful for efilter queries. """ - name = "agent_info" + name = "system_info" + mode = "mode_live" table_header = [ - dict(name="key"), + dict(name="key", width=20), dict(name="value") ] def collect(self): uname = UnameImpl.from_current_system(session=self.session) - for k, v in uname.to_primitive().iteritems(): + for k, v in uname.to_primitive(with_type=False).iteritems(): yield dict(key=k, value=v) diff --git a/rekall-core/rekall/plugins/common/efilter_plugins/ipython.py b/rekall-core/rekall/plugins/common/efilter_plugins/ipython.py index 637ddb352..9db1dbb64 100644 --- a/rekall-core/rekall/plugins/common/efilter_plugins/ipython.py +++ b/rekall-core/rekall/plugins/common/efilter_plugins/ipython.py @@ -40,6 +40,11 @@ def pager(self, line, cell=None): if " " in line: _, line_end = line.split(" ", 1) else: + # A bare pager magic with pager already set, means to clear it. + if session.GetParameter("pager"): + session.SetParameter("pager", None) + return + line_end = "less" session.SetParameter("pager", line_end) diff --git a/rekall-core/rekall/plugins/common/efilter_plugins/search.py b/rekall-core/rekall/plugins/common/efilter_plugins/search.py index dae876071..3a7121f2d 100644 --- a/rekall-core/rekall/plugins/common/efilter_plugins/search.py +++ b/rekall-core/rekall/plugins/common/efilter_plugins/search.py @@ -87,13 +87,13 @@ from efilter.protocols import applicative from efilter.protocols import associative -from efilter.protocols import counted from efilter.protocols import repeated from efilter.protocols import structured from rekall import obj from rekall import plugin from rekall import testlib +from rekall.plugins.response import common from rekall.plugins.overlays import basic from rekall.plugins.common.efilter_plugins import helpers from rekall.ui import identity as identity_renderer @@ -443,6 +443,14 @@ def _get_scopes(self): scopes["timestamp"] = api.user_func( lambda x, **_: basic.UnixTimeStamp(value=x, session=self.session), arg_types=[float, int, long]) + + # This function is used to indicate that the string represents + # a filename. This will cause the agent to upload it if the + # user requested uploading files. + # > select file(path.filename.name).filename.name from glob("/*") + scopes["file"] = api.user_func( + lambda x: common.FileInformation(session=self.session, filename=x), + arg_types=[unicode, str]) return scopes # IStructured implementation for EFILTER: diff --git a/rekall-core/rekall/plugins/overlays/basic.py b/rekall-core/rekall/plugins/overlays/basic.py index e20033ce4..480629f59 100644 --- a/rekall-core/rekall/plugins/overlays/basic.py +++ b/rekall-core/rekall/plugins/overlays/basic.py @@ -86,8 +86,12 @@ def v(self, vm=None): vm = vm or self.obj_vm data = vm.read(self.obj_offset, length) if self.term is not None: - left, sep, _ = data.partition(self.term) - data = left + sep + try: + left, sep, _ = data.partition(self.term) + data = left + sep + # We can not split it, just return the full length. + except ValueError: + pass return data diff --git a/rekall-core/rekall/plugins/response/common.py b/rekall-core/rekall/plugins/response/common.py index 8e43a4ee9..875d128cd 100644 --- a/rekall-core/rekall/plugins/response/common.py +++ b/rekall-core/rekall/plugins/response/common.py @@ -89,7 +89,8 @@ def __init__(self, filename, filesystem=u"API", path_sep=None): self.path_sep = path_sep or self.default_path_sep else: - raise TypeError("Filename must be a string or file spec.") + raise TypeError("Filename must be a string or file spec not %s." % type( + filename)) @property def dirname(self): diff --git a/rekall-core/rekall/plugins/response/files.py b/rekall-core/rekall/plugins/response/files.py index 92277cdb6..ac8aff66a 100644 --- a/rekall-core/rekall/plugins/response/files.py +++ b/rekall-core/rekall/plugins/response/files.py @@ -69,7 +69,7 @@ class IRStat(common.AbstractIRCommandPlugin): __args = [ dict(name="paths", positional=True, type="Array", - help="Paths to hash."), + help="Paths to stat."), ] table_header = [ @@ -123,8 +123,10 @@ def calculate_hashes(self, hashes, file_info): for hasher in hashers.values(): hasher.update(data) - return [Hash(type=name, value=hasher.digest()) - for name, hasher in hashers.iteritems()] + for key in list(hashers): + hashers[key] = hashers[key].hexdigest() + + return hashers def collect(self): for path in self.plugin_args.paths: diff --git a/rekall-core/rekall/plugins/response/osquery.py b/rekall-core/rekall/plugins/response/osquery.py index 332c9c9a9..193385de9 100644 --- a/rekall-core/rekall/plugins/response/osquery.py +++ b/rekall-core/rekall/plugins/response/osquery.py @@ -35,6 +35,9 @@ import platform import subprocess +from distutils import spawn + +from rekall import plugin from rekall import resources from rekall.plugins.response import common @@ -70,6 +73,10 @@ def try_to_find_osquery(self): if os.access(result, os.R_OK): return result + else: + # Try to find it somewhere on the system. + return spawn.find_executable("osqueryi") + raise e def render(self, renderer): @@ -79,6 +86,9 @@ def render(self, renderer): if osquery_path == None: osquery_path = self.try_to_find_osquery() + if not self.plugin_args.query: + raise plugin.PluginError("Query must be provided") + self.session.logging.debug("Found OSQuery at %s" % osquery_path) self.json_result = json.loads( subprocess.check_output( diff --git a/rekall-core/resources/rekall-agent-windows.bat b/rekall-core/resources/rekall-agent-windows.bat index 815c1562b..faed5c7be 100644 --- a/rekall-core/resources/rekall-agent-windows.bat +++ b/rekall-core/resources/rekall-agent-windows.bat @@ -1,3 +1,5 @@ +echo off + rem This batch script installs Rekall Agent as a windows service using rem the nssm tool which was originally downloaded from http://nssm.cc/ diff --git a/rekall-core/resources/rekall-agent.yaml b/rekall-core/resources/rekall-agent.yaml index 858e293c1..0ea0758db 100644 --- a/rekall-core/resources/rekall-agent.yaml +++ b/rekall-core/resources/rekall-agent.yaml @@ -7,7 +7,7 @@ client: # Add intrinsic labels to this client here. labels: - All - - Linux + - Windows # Set to false to prevent the agent from polling - it will just # exit after a single poll. Note that the nanny will wake it anyway @@ -23,4 +23,4 @@ client: base: http://127.0.0.1:8080/api/control path_prefix: /manifest # Persistent path to keep the same client. - writeback_path: rekall-agent-writeback.json \ No newline at end of file + writeback_path: rekall-agent-writeback.json diff --git a/rekall-core/setup.py b/rekall-core/setup.py index e8fac8b72..d697dc0e6 100755 --- a/rekall-core/setup.py +++ b/rekall-core/setup.py @@ -56,7 +56,7 @@ def find_data_files(source): "PyYAML == 3.11", "acora == 2.0", "arrow == 0.7.0", - "efilter == 1!1.5.2", + "rekall-efilter == 1.5.2", "intervaltree == 2.1.0", "pycrypto == 2.6.1", "pyelftools == 0.24", diff --git a/version.py b/version.py index 183a006cb..337cde952 100755 --- a/version.py +++ b/version.py @@ -5,7 +5,7 @@ This program is used to manage versions. Prior to each release, please run it with update. """ - +import arrow import argparse import json import os @@ -138,14 +138,24 @@ def get_versions(version_file="version.yaml"): def escape_string(instr): return instr.replace('"""', r'\"\"\"') +TEMPLATES = [ + "debian/changelog.in" +] + + +def update_templates(version_data): + version_data["debian_ts"] = arrow.utcnow().format( + 'ddd, D MMM YYYY h:mm:ss Z') + for path in TEMPLATES: + if not path.endswith(".in"): + continue + + target = path[:-3] + with open(target, "wb") as outfd: + outfd.write(open(path).read() % version_data) -def update(args): - if (args.version is None and - args.post is None and - args.rc is None and - args.codename is None): - raise AttributeError("You must set something in this release.") +def update_version_files(args): data, version_path = get_config_file(args.version_file) version_data = data["version_data"] if args.version: @@ -180,6 +190,18 @@ def update(args): with open(version_path, "wb") as fd: fd.write(contents) + update_templates(version_data) + + +def update(args): + if (args.version is None and + args.post is None and + args.rc is None and + args.codename is None): + raise AttributeError("You must set something in this release.") + + update_version_files(args) + def main(): parser = argparse.ArgumentParser()