From b9513b8e2a94dd5f5d89bb804eeb33071f545b86 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Fri, 14 Nov 2014 01:45:46 +0100 Subject: [PATCH] Fixed Bugs * All tests are now passing. * Added a TLB to the paging ASs to make them go faster. Review URL: https://codereview.appspot.com/167660044 --- .travis.yml | 7 + rekall/addrspace.py | 3 +- rekall/args.py | 179 +++++++++++-- rekall/config.py | 248 +++++++++--------- rekall/obj.py | 86 +++--- rekall/plugin.py | 97 +------ rekall/plugins/addrspaces/amd64.py | 7 +- rekall/plugins/addrspaces/crash.py | 3 + rekall/plugins/addrspaces/ewf.py | 17 +- rekall/plugins/addrspaces/intel.py | 91 +++++-- rekall/plugins/addrspaces/mips.py | 10 +- rekall/plugins/addrspaces/standard.py | 2 +- rekall/plugins/collectors/ballast.py | 2 +- rekall/plugins/common/address_resolver.py | 9 +- rekall/plugins/common/entities.py | 29 +- rekall/plugins/core.py | 16 +- rekall/plugins/guess_profile.py | 5 +- rekall/plugins/imagecopy.py | 2 +- rekall/plugins/overlays/basic.py | 8 +- rekall/plugins/overlays/windows/common.py | 50 +++- rekall/plugins/overlays/windows/pe_vtypes.py | 3 +- rekall/plugins/renderers/json_storage.py | 10 +- rekall/plugins/renderers/windows.py | 3 +- rekall/plugins/tools/json_tools.py | 55 ---- rekall/plugins/tools/profile_tool.py | 4 + rekall/plugins/windows/address_resolver.py | 53 ++-- rekall/plugins/windows/disassembler.py | 5 +- rekall/plugins/windows/gui/autodetect.py | 32 +-- rekall/plugins/windows/gui/userhandles.py | 8 +- rekall/plugins/windows/gui/win32k_core.py | 5 +- rekall/plugins/windows/handles.py | 1 - rekall/plugins/windows/interactive/structs.py | 1 + rekall/plugins/windows/kernel.py | 3 +- rekall/plugins/windows/kpcr.py | 6 +- rekall/plugins/windows/malware/apihooks.py | 14 +- rekall/plugins/windows/malware/callbacks.py | 139 +++++----- rekall/plugins/windows/modules.py | 2 +- rekall/plugins/windows/pagefile.py | 110 ++++---- rekall/plugins/windows/pas2kas.py | 6 +- rekall/plugins/windows/pfn.py | 52 ++-- rekall/plugins/windows/pfn_test.py | 14 + rekall/plugins/windows/ssdt.py | 15 +- rekall/plugins/windows/vadinfo.py | 4 +- rekall/plugins/windows/vadinfo_test.py | 8 + rekall/registry.py | 2 - rekall/rekal.py | 3 +- rekall/scan.py | 4 +- rekall/session.py | 11 +- rekall/testlib.py | 13 + rekall/ui/json_renderer.py | 4 +- rekall/ui/renderer.py | 6 +- rekall/ui/text.py | 20 +- setup.py | 2 +- tools/testing/test_suite.py | 1 + 54 files changed, 852 insertions(+), 638 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..31a66bdf2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: 2.7 +install: + - git clone https://github.com/scudette/rekall-test.git + - python setup.py install + +script: python tools/testing/test_suite.py -c rekall-test/tigger/tests.config diff --git a/rekall/addrspace.py b/rekall/addrspace.py index a42df5dc5..c315068fd 100755 --- a/rekall/addrspace.py +++ b/rekall/addrspace.py @@ -72,7 +72,7 @@ def __init__(self, base=None, session=None, write=False, profile=None, if session is None and base is not None: session = base.session - self.base = base or self + self.base = base self.profile = profile self.session = session if session is None: @@ -224,7 +224,6 @@ def __init__(self, base_offset=0, data='', **kwargs): super(BufferAddressSpace, self).__init__(**kwargs) self.fname = "Buffer" self.data = data - self.base = self self.base_offset = base_offset def assign_buffer(self, data, base_offset=0): diff --git a/rekall/args.py b/rekall/args.py index eaa5f2cdd..bf9b43e21 100644 --- a/rekall/args.py +++ b/rekall/args.py @@ -27,6 +27,7 @@ import argparse import logging +import re import os import sys import zipfile @@ -34,13 +35,14 @@ from rekall import config from rekall import constants from rekall import plugin +from rekall import utils -config.DeclareOption("--plugin", default=[], nargs="+", +config.DeclareOption("--plugin", default=[], type="ArrayStringParser", help="Load user provided plugin bundle.") config.DeclareOption( - "-h", "--help", default=False, action="store_true", + "-h", "--help", default=False, type="Boolean", help="Show help about global paramters.") @@ -167,14 +169,24 @@ def _TruncateARGV(argv): def ParseGlobalArgs(parser, argv, user_session): """Parse some session wide args which must be done before anything else.""" # Register global args. - config.RegisterArgParser(parser) + ConfigureCommandLineParser(config.OPTIONS, parser) # Parse the known args. known_args, _ = parser.parse_known_args(args=argv) with user_session.state as state: for arg, value in vars(known_args).items(): - state.Set(arg, value) + # Argparse tries to interpolate defaults into the parsed data in the + # event that the args are not present - even when calling + # parse_known_args. Before we get to this point, the config system + # has already set the state from the config file, so if we allow + # argparse to set the default we would override the config file + # (with the defaults). We solve this by never allowing argparse + # itself to handle the defaults. We always set default=None, when + # configuring the parser, and rely on the + # config.MergeConfigOptions() to set the defaults. + if value is not None: + state.Set(arg, value) # Enforce the appropriate logging level if user supplies the --verbose # or --quiet command line flags. @@ -193,7 +205,12 @@ def ParseGlobalArgs(parser, argv, user_session): # Now load the third party user plugins. These may introduce additional # plugins with args. - LoadPlugins(user_session.state.plugin) + if user_session.state.plugin: + LoadPlugins(user_session.state.plugin) + + # External files might have introduced new plugins - rebuild the plugin + # DB. + user_session.plugins.plugin_db.Rebuild() # Possibly restore the session from a file. session_filename = getattr(known_args, "session_filename", None) @@ -252,18 +269,42 @@ def ConfigureCommandLineParser(command_metadata, parser, critical=False): - Parsed from json from the web console. """ - group = parser.add_argument_group( - "Plugin %s options" % command_metadata.plugin_cls.name) + # This is used to allow the user to break the command line arbitrarily. + parser.add_argument('-', dest='__dummy', action="store_true", + help="A do nothing arg. Useful to separate options " + "which table multiple args from positional. Can be " + "specified many times.") + + try: + groups = parser.groups + except AttributeError: + groups = parser.groups = { + "None": parser.add_argument_group("Global options") + } + + if command_metadata.plugin_cls: + groups[command_metadata.plugin_cls.name] = parser.add_argument_group( + "Plugin %s options" % command_metadata.plugin_cls.name) for name, options in command_metadata.args.iteritems(): kwargs = options.copy() name = kwargs.pop("name", None) or name + kwargs.pop("default", None) + + group_name = kwargs.pop("group", None) + if group_name is None and command_metadata.plugin_cls: + group_name = command_metadata.plugin_cls.name + + group = groups.get(group_name) + if group is None: + groups[group_name] = group = parser.add_argument_group(group_name) positional_args = [] - short_opt = kwargs.pop("short_opt") + + short_opt = kwargs.pop("short_opt", None) # A positional arg is allows to be specified without a flag. - if kwargs.pop("positional"): + if kwargs.pop("positional", None): # By default positional args are required. required = kwargs.pop("required", True) @@ -282,11 +323,18 @@ def ConfigureCommandLineParser(command_metadata, parser, critical=False): arg_type = kwargs.pop("type", None) if arg_type == "ArrayIntParser": - kwargs["action"] = config.ArrayIntParser + kwargs["action"] = ArrayIntParser + kwargs["nargs"] = "+" + + if arg_type == "ArrayStringParser": + kwargs["action"] = ArrayStringParser kwargs["nargs"] = "+" elif arg_type == "IntParser": - kwargs["action"] = config.IntParser + kwargs["action"] = IntParser + + elif arg_type == "Float": + kwargs["type"] = float elif arg_type == "Boolean": kwargs["action"] = "store_true" @@ -294,6 +342,7 @@ def ConfigureCommandLineParser(command_metadata, parser, critical=False): # Multiple entries of choices (requires a choices paramter). elif arg_type == "ChoiceArray": kwargs["nargs"] = "+" + kwargs["action"] = ChoiceArrayParser # Skip option if not critical. critical_arg = kwargs.pop("critical", False) @@ -310,11 +359,6 @@ def parse_args(argv=None, user_session=None): if argv is None: argv = sys.argv[1:] - # The plugin name is taken from the command line, but it is not enough to - # know which specific implementation will be used. For example there are 3 - # classes implementing the pslist plugin WinPsList, LinPsList and OSXPsList. - plugin_name, argv = FindPlugin(argv, user_session) - parser = RekallArgParser( description=constants.BANNER, conflict_handler='resolve', @@ -322,6 +366,14 @@ def parse_args(argv=None, user_session=None): epilog="When no module is provided, drops into interactive mode", formatter_class=RekallHelpFormatter) + # Parse the global and critical args from the command line. + ParseGlobalArgs(parser, argv, user_session) + + # The plugin name is taken from the command line, but it is not enough to + # know which specific implementation will be used. For example there are 3 + # classes implementing the pslist plugin WinPsList, LinPsList and OSXPsList. + plugin_name, argv = FindPlugin(argv, user_session) + # Add all critical parameters. Critical parameters are those which are # common to all implementations of a certain plugin and are required in # order to choose from these implementations. For example, the profile or @@ -330,9 +382,6 @@ def parse_args(argv=None, user_session=None): for metadata in user_session.plugins.plugin_db.MetadataByName(plugin_name): ConfigureCommandLineParser(metadata, parser, critical=True) - # Parse the global and critical args from the command line. - ParseGlobalArgs(parser, argv, user_session) - # Find the specific implementation of the plugin that applies here. For # example, we have 3 different pslist implementations depending on the # specific profile loaded. @@ -353,4 +402,96 @@ def parse_args(argv=None, user_session=None): parser.print_help() sys.exit(-1) + # Apply the defaults to the parsed args. + result = utils.AttributeDict(vars(result)) + result.pop("__dummy", None) + + command_metadata.ApplyDefaults(result) + return plugin_cls, result + + +## Parser for special args. + +class IntParser(argparse.Action): + """Class to parse ints either in hex or as ints.""" + def parse_int(self, value): + # Support suffixes + multiplier = 1 + m = re.search("(.*)(mb|kb|m|k)", value) + if m: + value = m.group(1) + suffix = m.group(2).lower() + if suffix in ("mb", "m"): + multiplier = 1024 * 1024 + elif suffix in ("kb", "k"): + multiplier = 1024 + + try: + if value.startswith("0x"): + value = int(value, 16) * multiplier + else: + value = int(value) * multiplier + except ValueError: + raise argparse.ArgumentError(self, "Invalid integer value") + + return value + + def __call__(self, parser, namespace, values, option_string=None): + if isinstance(values, basestring): + values = self.parse_int(values) + setattr(namespace, self.dest, values) + + +class ArrayIntParser(IntParser): + """Parse input as a comma separated list of integers. + + We support input in the following forms: + + --pid 1,2,3,4,5 + + --pid 1 2 3 4 5 + + --pid 0x1 0x2 0x3 + """ + + def Validate(self, value): + return self.parse_int(value) + + def __call__(self, parser, namespace, values, option_string=None): + result = [] + if isinstance(values, basestring): + values = [values] + + for value in values: + result.extend([self.Validate(x) for x in value.split(",")]) + + setattr(namespace, self.dest, result) + + +class ChoiceArrayParser(ArrayIntParser): + + def __init__(self, *args, **kwargs): + self._choices = kwargs.pop("choices", []) + super(ChoiceArrayParser, self).__init__(*args, **kwargs) + + def Validate(self, value): + if value not in self._choices: + raise argparse.ArgumentError( + None, "Choice %r not valid. Valid choices are %s" % ( + value, self._choices)) + + return value + + +class ArrayStringParser(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + result = [] + + if isinstance(values, basestring): + values = [values] + + for value in values: + result.extend([x for x in value.split(",")]) + + setattr(namespace, self.dest, result) diff --git a/rekall/config.py b/rekall/config.py index 757af454a..a67978e2f 100644 --- a/rekall/config.py +++ b/rekall/config.py @@ -31,14 +31,119 @@ __author__ = "Michael Cohen " -import argparse +import collections import logging -import re import yaml import os from rekall import constants + +class CommandMetadata(object): + """A class that carried a plugin's configuration. + + A plugin is responsible for declaring its metadata by calling this + configuration object's methods from the args() class method. + + There are two things that plugin must declare: + + add_*_arg(): Calling these functions declares an argument for this + plugin. See the documentation for that method for details. + + add_metadata(): This method provides additional metadata about this plugin. + """ + + def __init__(self, plugin_cls=None): + self.args = collections.OrderedDict() + self.requirements = set() + self.plugin_cls = plugin_cls + if plugin_cls: + plugin_cls.args(self) + + self.description = (plugin_cls.__doc__ or + plugin_cls.__init__.__doc__ or "") + + def set_description(self, description): + self.description = description + + def add_positional_arg(self, name, type="string"): + """Declare a positional arg.""" + self.args[name] = dict(type=type) + + def add_argument(self, short_opt, long_opt=None, **options): + """Add a new argument to the command. + + This method is used in the args() class method to add a new command line + arg to the plugin. It is similar to the argparse add_argument() method + but it adds a type parameter which conveys higher level information + about the argument. Currently supported types: + + - ArrayIntParser: A list of integers (possibly encoded as hex strings). + - ArrayStringParser: A list of strings. + - Float: A float. + - IntParser: An integer (possibly encoded as a hex string). + - Boolean: A flag - true/false. + - ChoiceArray: A comma separated list of strings which must be from the + choices parameter. + """ + if "action" in options: + raise RuntimeError("Action keyword is deprecated.") + + if not isinstance(options.get("type", ""), str): + raise RuntimeError("Type must be a string.") + + # Is this a positional arg? + positional = options.pop("positional", True) + + # For now we support option names with leading --. + if long_opt is None: + long_opt = short_opt + short_opt = "" + + if long_opt.startswith("-"): + long_opt = long_opt.lstrip("-") + short_opt = short_opt.lstrip("-") + positional = False + + name = long_opt + options["short_opt"] = short_opt + options["positional"] = positional + options["name"] = name + + self.args[name] = options + + def add_requirement(self, requirement): + """Add a requirement for this plugin. + + Currently supported requirements: + - profile: A profile must exist for this plugin to run. + + - physical_address_space: A Physical Address Space (i.e. an image file) + must exist for this plugin to work. + """ + self.requirements.add(requirement) + + def Metadata(self): + return dict(requirements=list(self.requirements), + arguments=self.args.values(), name=self.plugin_cls.name, + description=self.description) + + def ApplyDefaults(self, args): + """Update args with the defaults. + + If an option in args is None, we update it with the default value for + this option. + """ + for name, options in self.args.iteritems(): + if options.get("dest") == "SUPPRESS": + continue + + name = name.replace("-", "_") + if args[name] is None: + args[name] = options.get("default") + + return args + def GetHomeDir(): return (os.environ.get("HOME") or # Unix os.environ.get("USERPROFILE")) # Windows @@ -54,8 +159,9 @@ def GetHomeDir(): notebook_dir=GetHomeDir(), ) - -OPTIONS = [] +# Global options control the framework's own flags. They are not associated with +# any one plugin. +OPTIONS = CommandMetadata() def GetConfigFile(): @@ -89,11 +195,6 @@ def GetConfigFile(): def MergeConfigOptions(state): """Read the config file and apply the config options to the session.""" - # First apply the defaults: - for _, _, name, default, _ in OPTIONS: - if default is not None: - state.Set(name, default) - config_data = GetConfigFile() # An empty configuration file - we try to initialize a new one. if not config_data: @@ -113,6 +214,11 @@ def MergeConfigOptions(state): if not config_data: config_data = DEFAULT_CONFIGURATION + # First apply the defaults: + for name, options in OPTIONS.args.iteritems(): + if name not in config_data: + config_data[name] = options.get("default") + for k, v in config_data.items(): state.Set(k, v) @@ -121,126 +227,14 @@ def RemoveGlobalOptions(state): """Remove all global options from state dictionary.""" state.pop("SUPPRESS", None) - for _, _, name, _, _ in OPTIONS: + for name in OPTIONS.args: state.pop(name, None) return state -def DeclareOption(short_name=None, name=None, default=None, group=None, - **kwargs): - """Declare a config option for command line and config file. - - Arguments: - short_name: The one-letter name of the flag (like -v). - name: The long name of the flag (like --verbose). - default: The default value. - group: Arguments from the same group are rendered together. - - The remaining keyword arguments are passed on to the argument parser - (see RekallArgParser). - """ - if name is None: - name = short_name - short_name = None - - name = name.strip("-") - if short_name: - short_name = short_name.strip("-") - - OPTIONS.append((group, short_name, name, default, kwargs)) - - -def RegisterArgParser(parser): - """Register the options into the parser.""" - groups = {} - - for group, short_name, name, _, kwargs in sorted(OPTIONS): - if not name.startswith("--"): - name = "--" + name - - if short_name and not short_name.startswith("-"): - short_name = "-" + short_name - - kwargs["default"] = argparse.SUPPRESS - if group: - try: - arg_group = groups[group] - except KeyError: - groups[group] = arg_group = parser.add_argument_group(group) - - if short_name: - arg_group.add_argument(short_name, name, **kwargs) - else: - arg_group.add_argument(name, **kwargs) - - else: - if short_name: - parser.add_argument(short_name, name, **kwargs) - else: - parser.add_argument(name, **kwargs) - - -class IntParser(argparse.Action): - """Class to parse ints either in hex or as ints.""" - def parse_int(self, value): - # Support suffixes - multiplier = 1 - m = re.search("(.*)(mb|kb|m|k)", value) - if m: - value = m.group(1) - suffix = m.group(2).lower() - if suffix in ("mb", "m"): - multiplier = 1024 * 1024 - elif suffix in ("kb", "k"): - multiplier = 1024 - - try: - if value.startswith("0x"): - value = int(value, 16) * multiplier - else: - value = int(value) * multiplier - except ValueError: - raise argparse.ArgumentError(self, "Invalid integer value") - - return value - - def __call__(self, parser, namespace, values, option_string=None): - if isinstance(values, basestring): - values = self.parse_int(values) - setattr(namespace, self.dest, values) - - -class ArrayIntParser(IntParser): - """Parse input as a comma separated list of integers. - - We support input in the following forms: - - --pid 1,2,3,4,5 - - --pid 1 2 3 4 5 - - --pid 0x1 0x2 0x3 - """ - - def __call__(self, parser, namespace, values, option_string=None): - result = [] - if isinstance(values, basestring): - values = [values] - - for value in values: - result.extend([self.parse_int(x) for x in value.split(",")]) - - setattr(namespace, self.dest, result) - - -class ArrayStringParser(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - result = [] - if isinstance(values, basestring): - values = [values] - - for value in values: - result.extend([x for x in value.split(",")]) - - setattr(namespace, self.dest, result) +def DeclareOption(*args, **kwargs): + """Declare a config option for command line and config file.""" + # Options can not be positional! + kwargs["positional"] = False + OPTIONS.add_argument(*args, **kwargs) diff --git a/rekall/obj.py b/rekall/obj.py index 4de6c8bb4..8cac0fcf8 100644 --- a/rekall/obj.py +++ b/rekall/obj.py @@ -325,8 +325,8 @@ def __init__(self, type_name=None, offset=0, vm=None, profile=None, the vtype language definition. """ if kwargs: - logging.error("Unknown keyword args {0} for {1}".format( - kwargs, self.__class__.__name__)) + logging.error("Unknown keyword args %s for %s", + kwargs, self.__class__.__name__) self.obj_type = type_name @@ -341,7 +341,7 @@ def __init__(self, type_name=None, offset=0, vm=None, profile=None, self.obj_session = session if profile is None: - logging.critical("Profile must be provided to %s" % self) + logging.critical("Profile must be provided to %s", self) raise RuntimeError("Profile must be provided") @property @@ -389,8 +389,7 @@ def __nonzero__(self): the later form is not going to work when X is a NoneObject. """ - result = self.obj_vm.is_valid_address(self.obj_offset) - return result + return self.is_valid() def __eq__(self, other): return self.v() == other or ( @@ -417,10 +416,10 @@ def indices(self): return (self.v(),) def m(self, memname): - return NoneObject("No member {0}".format(memname)) + return NoneObject("No member {0}", memname) def is_valid(self): - return self.obj_vm.is_valid_address(self.obj_offset) + return True def deref(self, vm=None): """An alias for dereference - less to type.""" @@ -428,8 +427,7 @@ def deref(self, vm=None): def dereference(self, vm=None): _ = vm - return NoneObject("Can't dereference {0}".format( - self.obj_name), self.obj_profile) + return NoneObject("Can't dereference {0}", self.obj_name) def reference(self): """Produces a pointer to this object. @@ -457,8 +455,7 @@ def v(self, vm=None): allow for this. """ _ = vm - return NoneObject("No value for {0}".format( - self.obj_name), self.obj_profile) + return NoneObject("No value for {0}", self.obj_name) def __str__(self): return utils.SmartStr(self) @@ -585,13 +582,13 @@ def v(self, vm=None): data = self.obj_vm.read(self.obj_offset, self.obj_size) if not data: - return NoneObject("Unable to read {0} bytes from {1}".format( - self.obj_size, self.obj_offset)) + return NoneObject("Unable to read {0} bytes from {1}", + self.obj_size, self.obj_offset) - (val,) = struct.unpack(self.format_string, data) + # Cache this for next time. + (self.value,) = struct.unpack(self.format_string, data) - self.value = val - return val + return self.value def cdecl(self): return self.obj_name @@ -705,7 +702,8 @@ def __eq__(self, other): def is_valid(self): """ Returns if what we are pointing to is valid """ - return self.obj_vm.is_valid_address(self.v()) + # Null pointers are invalid. + return self.v() != 0 def __getitem__(self, item): """Indexing a pointer indexes its target. @@ -749,10 +747,11 @@ def dereference(self, vm=None): # Target not valid, return void. result = Void(**kwargs) - return result - else: - return NoneObject("Pointer {0} @ {1} invalid".format( - self.obj_name, self.v())) + if result.is_valid(): + return result + + return NoneObject("Pointer {0} @ {1} invalid", + self.obj_name, self.v()) def __dir__(self): return dir(self.dereference()) @@ -1011,9 +1010,6 @@ def __getitem__(self, pos): return [self[i] for i in xrange(start, stop, step)] offset = self.target_size * pos + self.obj_offset - if not self.obj_vm.is_valid_address(offset): - return NoneObject("Invalid offset %s" % offset) - return self.obj_profile.Object( self.target, offset=offset, vm=self.obj_vm, parent=self, profile=self.obj_profile, @@ -1280,7 +1276,6 @@ def m(self, attr): To access a field which has been renamed in different OS versions. """ - ACCESS_LOG.LogFieldAccess(self.obj_profile.name, self.obj_type, attr) result = self._cache.get(attr) if result is not None: return result @@ -1293,17 +1288,17 @@ def m(self, attr): self._cache[attr] = result return result - if attr in self.members: + element = self.members.get(attr) + if element is not None: # Allow the element to be a callable rather than a list - this is # useful for aliasing member names - element = self.members[attr] if callable(element): return element(self) offset, cls = element else: - return NoneObject(u"Struct {0} has no member {1}".format( - self.obj_name, attr)) + return NoneObject(u"Struct {0} has no member {1}", + self.obj_name, attr) if callable(offset): ## If offset is specified as a callable its an absolute @@ -1498,7 +1493,7 @@ def Initialize(cls, profile): def __init__(self, name=None, session=None, metadata=None, **kwargs): if kwargs: - logging.error("Unknown keyword args {0}".format(kwargs)) + logging.error("Unknown keyword args %s", kwargs) if name is None: name = self.__class__.__name__ @@ -1733,9 +1728,9 @@ def compile_type(self, type_name): elif v[0] == None: logging.warning( - "{0} has no offset in object {1}. Check that vtypes " - "has a concrete definition for it.".format( - k, type_name)) + "%s has no offset in object %s. Check that vtypes " + "has a concrete definition for it.", + k, type_name) else: members[k] = (v[0], self.list_to_type(k, v[1])) @@ -2142,7 +2137,7 @@ def __getattr__(self, attr): return Curry(self.Object, attr) def Object(self, type_name=None, offset=None, vm=None, name=None, - parent=None, context=None, session=None, **kwargs): + parent=None, context=None, **kwargs): """ A function which instantiates the object named in type_name (as a string) from the type in profile passing optional args of kwargs. @@ -2164,18 +2159,16 @@ def Object(self, type_name=None, offset=None, vm=None, name=None, parent: The object can maintain a reference to its parent object. """ name = name or type_name - if session is None: - session = self.session # Ensure we are called correctly. - if not isinstance(name, basestring): + if name.__class__ not in (unicode, str): raise ValueError("Type name must be a string") if offset is None: offset = 0 if vm is None: vm = addrspace.BaseAddressSpace.classes["DummyAddressSpace"]( - size=self.get_obj_size(name) or 0, session=session) + size=self.get_obj_size(name) or 0, session=self.session) else: offset = int(offset) @@ -2189,11 +2182,12 @@ def Object(self, type_name=None, offset=None, vm=None, name=None, # If the cache contains a None, this member is not represented by a # vtype (it might be a pure object class or a constant). - if self.types[type_name] is not None: - result = self.types[type_name]( - offset=offset, vm=vm, name=name, - parent=parent, context=context, - session=session, **kwargs) + cls = self.types[type_name] + if cls is not None: + result = cls(offset=offset, vm=vm, name=name, + parent=parent, context=context, + session=self.session, **kwargs) + return result elif type_name in self.object_classes: @@ -2204,7 +2198,7 @@ def Object(self, type_name=None, offset=None, vm=None, name=None, name=name, parent=parent, context=context, - session=session, + session=self.session, **kwargs) if isinstance(result, Struct): @@ -2217,8 +2211,8 @@ def Object(self, type_name=None, offset=None, vm=None, name=None, else: # If we get here we have no idea what the type is supposed to be? - return NoneObject("Cant find object {0} in profile {1}?".format( - type_name, self)) + return NoneObject("Cant find object %s in profile %s?", + type_name, self) def __unicode__(self): return u"<%s profile %s (%s)>" % ( diff --git a/rekall/plugin.py b/rekall/plugin.py index 5f3c8f581..fd13939aa 100644 --- a/rekall/plugin.py +++ b/rekall/plugin.py @@ -21,9 +21,10 @@ __author__ = "Michael Cohen " -import collections + import StringIO +from rekall import config from rekall import obj from rekall import registry from rekall.ui import text as text_renderer @@ -41,92 +42,6 @@ class InvalidArgs(Error): """Invalid arguments.""" -class CommandMetadata(object): - """A class that carried a plugin's configuration. - - A plugin is responsible for declaring its metadata by calling this - configuration object's methods from the args() class method. - - There are two things that plugin must declare: - - add_*_arg(): Calling these functions declares an argument for this - plugin. See the documentation for that method for details. - - add_metadata(): This method provides additional metadata about this plugin. - """ - - def __init__(self, plugin_cls): - self.args = collections.OrderedDict() - self.requirements = set() - self.plugin_cls = plugin_cls - plugin_cls.args(self) - self.description = (plugin_cls.__doc__ or - plugin_cls.__init__.__doc__ or "") - - def set_description(self, description): - self.description = description - - def add_positional_arg(self, name, type="string"): - """Declare a positional arg.""" - self.args[name] = dict(type=type) - - def add_argument(self, short_opt, long_opt=None, **options): - """Add a new argument to the command. - - This method is used in the args() class method to add a new command line - arg to the plugin. It is similar to the argparse add_argument() method - but it adds a type parameter which conveys higher level information - about the argument. Currently supported types: - - - ArrayIntParser: A list of integers (possibly encoded as hex strings). - - IntParser: An integer (possibly encoded as a hex string). - - Boolean: A flag - true/false. - - ChoiceArray: A comma separated list of strings which must be from the - choices parameter. - """ - if "action" in options: - raise RuntimeError("Action keyword is deprecated.") - - if not isinstance(options.get("type", ""), str): - raise RuntimeError("Type must be a string.") - - # Is this a positional arg? - positional = True - - # For now we support option names with leading --. - if long_opt is None: - long_opt = short_opt - short_opt = "" - - if long_opt.startswith("-"): - long_opt = long_opt.lstrip("-") - short_opt = short_opt.lstrip("-") - positional = False - - name = long_opt - options["short_opt"] = short_opt - options["positional"] = positional - options["name"] = name - - self.args[name] = options - - def add_requirement(self, requirement): - """Add a requirement for this plugin. - - Currently supported requirements: - - profile: A profile must exist for this plugin to run. - - - physical_address_space: A Physical Address Space (i.e. an image file) - must exist for this plugin to work. - """ - self.requirements.add(requirement) - - def Metadata(self): - return dict(requirements=list(self.requirements), - arguments=self.args.values(), name=self.plugin_cls.name, - description=self.description) - - class Command(object): """A command can be run from the rekall command line. @@ -402,12 +317,16 @@ class PluginMetadataDatabase(object): """A database of all the currently registered plugin's metadata.""" def __init__(self, session): - self.db = {} self.session = session + self.Rebuild() + + def Rebuild(self): + self.db = {} + for plugin_cls in Command.classes.itervalues(): plugin_name = plugin_cls.name self.db.setdefault(plugin_name, []).append( - CommandMetadata(plugin_cls)) + config.CommandMetadata(plugin_cls)) def MetadataByName(self, name): """Return all Implementations that implement command name.""" diff --git a/rekall/plugins/addrspaces/amd64.py b/rekall/plugins/addrspaces/amd64.py index 1bc4b9490..ab260e76b 100644 --- a/rekall/plugins/addrspaces/amd64.py +++ b/rekall/plugins/addrspaces/amd64.py @@ -22,6 +22,7 @@ """ This is based on Jesse Kornblum's patch to clean up the standard AS's. """ +# pylint: disable=protected-access import struct @@ -29,8 +30,8 @@ from rekall.plugins.addrspaces import intel -config.DeclareOption(name="ept", group="Virtualization support", - action=config.ArrayIntParser, +config.DeclareOption("ept", group="Virtualization support", + type="ArrayIntParser", help="The EPT physical address.") @@ -247,7 +248,7 @@ def __init__(self, ept=None, **kwargs): # A dummy DTB is passed to the base class so the DTB checks on # IA32PagedMemory don't bail out. We require the DTB to never be used # for page translation outside of get_pml4e. - AMD64PagedMemory.__init__(self, dtb=0xFFFFFFFF, **kwargs) + super(VTxPagedMemory, self).__init__(dtb=0xFFFFFFFF, **kwargs) # Reset the DTB, in case a plugin or AS relies on us providing one. self.dtb = None diff --git a/rekall/plugins/addrspaces/crash.py b/rekall/plugins/addrspaces/crash.py index 5724715be..7d13b0c7e 100644 --- a/rekall/plugins/addrspaces/crash.py +++ b/rekall/plugins/addrspaces/crash.py @@ -41,6 +41,9 @@ class WindowsCrashDumpSpace32(addrspace.RunBasedAddressSpace): def __init__(self, **kwargs): super(WindowsCrashDumpSpace32, self).__init__(**kwargs) + + self.as_assert(self.base != None, "No base address space provided") + self.offset = 0 self.fname = '' diff --git a/rekall/plugins/addrspaces/ewf.py b/rekall/plugins/addrspaces/ewf.py index 120c9f219..c77cd845e 100644 --- a/rekall/plugins/addrspaces/ewf.py +++ b/rekall/plugins/addrspaces/ewf.py @@ -39,20 +39,25 @@ class EWFAddressSpace(addrspace.BaseAddressSpace): order = 20 __image = True - def __init__(self, base=None, **kwargs): + def __init__(self, **kwargs): super(EWFAddressSpace, self).__init__(**kwargs) # Fail quickly if this is not an EWF file. - self.as_assert(base != None, "No base address space provided") + self.as_assert(self.base != None, "No base address space provided") - self.as_assert(base.read(0, 6) == "\x45\x56\x46\x09\x0D\x0A", + self.as_assert(self.base.read(0, 6) == "\x45\x56\x46\x09\x0D\x0A", "EWF signature not present") # Now try to open it as an ewf file. - self.ewf_file = ewf.EWFFile(session=self.session, address_space=base) + self.ewf_file = ewf.EWFFile( + session=self.session, address_space=self.base) def read(self, offset, length): - return self.ewf_file.read(offset, length) + res = self.ewf_file.read(offset, length) + if len(res) < length: + res += "\x00" * (length - len(res)) - def get_available_addresses(self): + return res + + def get_available_addresses(self, start=0): yield (0, 0, self.ewf_file.size) diff --git a/rekall/plugins/addrspaces/intel.py b/rekall/plugins/addrspaces/intel.py index acfb2cca1..81eef58d1 100644 --- a/rekall/plugins/addrspaces/intel.py +++ b/rekall/plugins/addrspaces/intel.py @@ -27,11 +27,33 @@ from rekall import addrspace from rekall import config +from rekall import utils +config.DeclareOption( + "dtb", group="Autodetection Overrides", + type="IntParser", help="The DTB physical address.") -config.DeclareOption(name="dtb", group="Autodetection Overrides", - action=config.IntParser, - help="The DTB physical address.") + +PAGE_SHIFT = 12 +PAGE_MASK = ~ 0xFFF + + +class TranslationLookasideBuffer(utils.FastStore): + """An implementation of a TLB.""" + + def Get(self, vaddr): + result = super(TranslationLookasideBuffer, self).Get( + vaddr >> PAGE_SHIFT) + + if result is not None: + return result + (vaddr & 0xFFF) + + def Put(self, vaddr, paddr): + if paddr is not None: + paddr = paddr & PAGE_MASK + + super(TranslationLookasideBuffer, self).Put( + vaddr >> PAGE_SHIFT, paddr) class IA32PagedMemory(addrspace.PagedReader): @@ -67,7 +89,7 @@ def __init__(self, name=None, dtb=None, **kwargs): super(IA32PagedMemory, self).__init__(**kwargs) ## We must be stacked on someone else: - if not self.base != self: + if not self.base: raise TypeError("No base Address Space") # If the underlying address space already knows about the dtb we use it. @@ -79,6 +101,9 @@ def __init__(self, name=None, dtb=None, **kwargs): " plugin to search for the dtb.") self.name = (name or 'Kernel AS') + "@%#x" % self.dtb + # Use a TLB to make this faster. + self._tlb = TranslationLookasideBuffer(1000) + def entry_present(self, entry): ''' Returns whether or not the 'P' (Present) flag is on @@ -149,18 +174,25 @@ def vtop(self, vaddr): The function should return either None (no valid mapping) or the offset in physical memory where the address maps. ''' - pde_value = self.get_pde(vaddr) - if not self.entry_present(pde_value): - return None + try: + return self._tlb.Get(vaddr) + except KeyError: + pde_value = self.get_pde(vaddr) + if not self.entry_present(pde_value): + return None - if self.page_size_flag(pde_value): - return self.get_four_meg_paddr(vaddr, pde_value) + if self.page_size_flag(pde_value): + return self.get_four_meg_paddr(vaddr, pde_value) - pte_value = self.get_pte(vaddr, pde_value) - if not self.entry_present(pte_value): - return None + pte_value = self.get_pte(vaddr, pde_value) + if not self.entry_present(pte_value): + return None + + res = self.get_phys_addr(vaddr, pte_value) + + self._tlb.Put(vaddr, res) + return res - return self.get_phys_addr(vaddr, pte_value) def read_long_phys(self, addr): ''' @@ -272,7 +304,6 @@ def get_pde(self, vaddr, pdpte): pde_addr = (pdpte & 0xffffffffff000) | ((vaddr & 0x3fe00000) >> 18) return self.read_long_long_phys(pde_addr) - def get_two_meg_paddr(self, vaddr, pde): ''' Return the offset in a 2MB memory page from the given virtual @@ -314,23 +345,29 @@ def vtop(self, vaddr): The function returns either None (no valid mapping) or the offset in physical memory where the address maps. ''' - pdpte = self.get_pdpte(vaddr) - if not self.entry_present(pdpte): - # Add support for paged out PDPTE - # Insert buffalo here! - return None + try: + return self._tlb.Get(vaddr) + except KeyError: + pdpte = self.get_pdpte(vaddr) + if not self.entry_present(pdpte): + # Add support for paged out PDPTE + # Insert buffalo here! + return None - pde = self.get_pde(vaddr, pdpte) - if not self.entry_present(pde): - # Add support for paged out PDE - return None + pde = self.get_pde(vaddr, pdpte) + if not self.entry_present(pde): + # Add support for paged out PDE + return None + + if self.page_size_flag(pde): + return self.get_two_meg_paddr(vaddr, pde) - if self.page_size_flag(pde): - return self.get_two_meg_paddr(vaddr, pde) + pte = self.get_pte(vaddr, pde) - pte = self.get_pte(vaddr, pde) + res = self.get_phys_addr(vaddr, pte) - return self.get_phys_addr(vaddr, pte) + self._tlb.Put(vaddr, res) + return res def read_long_long_phys(self, addr): ''' diff --git a/rekall/plugins/addrspaces/mips.py b/rekall/plugins/addrspaces/mips.py index 679afcdd7..d3e0200aa 100644 --- a/rekall/plugins/addrspaces/mips.py +++ b/rekall/plugins/addrspaces/mips.py @@ -18,13 +18,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # from rekall import addrspace -from rekall import config -from rekall import obj import struct -config.DeclareOption(name="dtb", group="Autodetection Overrides", - action=config.IntParser, - help="The DTB physical address.") pointer_size = 4 page_shift = 12 @@ -104,11 +99,12 @@ def get_pte(self, vaddr, pgd): return self.read_long_phys(pgd_val) def get_paddr(self, vaddr, pte): - return (self.pte_pfn(pte) << page_shift) | (vaddr & ((1 << page_shift) - 1)) + return ((self.pte_pfn(pte) << page_shift) | + (vaddr & ((1 << page_shift) - 1))) def entry_present(self, entry): if entry: - if (entry & 1): + if entry & 1: return True return False diff --git a/rekall/plugins/addrspaces/standard.py b/rekall/plugins/addrspaces/standard.py index ca726c4b8..622d63873 100644 --- a/rekall/plugins/addrspaces/standard.py +++ b/rekall/plugins/addrspaces/standard.py @@ -69,7 +69,7 @@ def read_long(self, addr): (longval,) = struct.unpack('=I', string) return longval - def get_available_addresses(self): + def get_available_addresses(self, start=0): yield (0, 0, self.fsize) def is_valid_address(self, addr): diff --git a/rekall/plugins/collectors/ballast.py b/rekall/plugins/collectors/ballast.py index 18dfbb670..413400a42 100644 --- a/rekall/plugins/collectors/ballast.py +++ b/rekall/plugins/collectors/ballast.py @@ -31,7 +31,7 @@ config.DeclareOption( - "--generate_ballast", default=0, action=config.IntParser, + "--generate_ballast", default=0, type="IntParser", help="If specified, fill the entity database with this many fake entries.") diff --git a/rekall/plugins/common/address_resolver.py b/rekall/plugins/common/address_resolver.py index 5d6609bf6..629f595cc 100644 --- a/rekall/plugins/common/address_resolver.py +++ b/rekall/plugins/common/address_resolver.py @@ -22,9 +22,16 @@ import re +from rekall import config from rekall import obj +config.DeclareOption( + "--name_resolution_strategies", default=["Module", "Symbol", "Export"], + group="Interface", type="ChoiceArray", + choices=["Module", "Symbol", "Export"]) + + class AddressResolverMixin(object): """The basic building block for constructing an address resolver plugin.""" @@ -124,7 +131,7 @@ def get_address_by_name(self, name): try: return int(name) - except ValueError: + except (ValueError, TypeError): pass if not isinstance(name, basestring): diff --git a/rekall/plugins/common/entities.py b/rekall/plugins/common/entities.py index 854a20501..e10307a3d 100644 --- a/rekall/plugins/common/entities.py +++ b/rekall/plugins/common/entities.py @@ -25,6 +25,7 @@ from rekall import config from rekall import plugin +from rekall import testlib from rekall.entities import component as entity_component @@ -32,6 +33,12 @@ from rekall.entities.query import query as entity_query +class TestListEvents(testlib.SortedComparison): + PARAMETERS = dict( + commandline="list_events" + ) + + class ListEvents(plugin.Command): __name = "list_events" @@ -75,6 +82,20 @@ def render(self, renderer): help="Filter to apply to all plugins backed by the entity layer.") +class TestEntityFind(testlib.SimpleTestCase): + PARAMETERS = dict( + commandline='find "Process/command =~ %(process)s"', + process="lsass" + ) + + + +class TestEntityAnalyze(testlib.SortedComparison): + PARAMETERS = dict( + commandline='analyze "Process/command =~ lsass"' + ) + + class EntityAnalyze(plugin.Command): __name = "analyze" @@ -83,7 +104,8 @@ class EntityAnalyze(plugin.Command): @classmethod def args(cls, parser): super(EntityAnalyze, cls).args(parser) - parser.add_positional_arg("query") + parser.add_argument("query", positional=True, + help="The filter query to use") def __init__(self, query=None, **kwargs): super(EntityAnalyze, self).__init__(**kwargs) @@ -185,11 +207,14 @@ class EntityFind(plugin.Command): @classmethod def args(cls, parser): super(EntityFind, cls).args(parser) - parser.add_positional_arg("query") + parser.add_argument("query", positional=True, + help="The filter query to use") + parser.add_argument("--components", default=None, nargs="+", type="str") parser.add_argument("--attributes", default=None, nargs="+", type="str") + parser.add_argument("--explain", type="Boolean", default=False, help="Show which part of the query matched.") diff --git a/rekall/plugins/core.py b/rekall/plugins/core.py index a0b2a0c6e..26bcd1ad1 100644 --- a/rekall/plugins/core.py +++ b/rekall/plugins/core.py @@ -319,7 +319,7 @@ def __init__(self, pas_spec="auto", **kwargs): # Parse Address spaces from this specification. TODO: Support EPT # specification and nesting. - ADDRESS_SPACE_RE = re.compile("([a-zA-Z0-9]+)@(0x[0-9a-zA-Z]+)") + ADDRESS_SPACE_RE = re.compile("([a-zA-Z0-9]+)@((0x)?[0-9a-zA-Z]+)") def ResolveAddressSpace(self, name=None): """Resolve the name into an address space. @@ -333,6 +333,8 @@ def ResolveAddressSpace(self, name=None): as_type@dtb_address: Instantiates the address space at the specified DTB. For example: amd64@0x18700 + + pid@pid_number: Use the process address space for the specified pid. """ if name is None: result = self.session.GetParameter("default_address_space") @@ -355,9 +357,16 @@ def ResolveAddressSpace(self, name=None): m = self.ADDRESS_SPACE_RE.match(name) if m: + arg = int(m.group(2), 0) + if m.group(1) == "pid": + for task in self.session.plugins.pslist( + pid=arg).filter_processes(): + return task.get_process_address_space() + raise AttributeError("Process pid %s not found" % arg) + as_cls = addrspace.BaseAddressSpace.classes.get(m.group(1)) if as_cls: - return as_cls(session=self.session, dtb=int(m.group(2), 0), + return as_cls(session=self.session, dtb=arg, base=self.GetPhysicalAddressSpace()) raise AttributeError("Address space specification %r invalid.", name) @@ -476,8 +485,9 @@ def GuessAddressSpace(self, base_as=None, **kwargs): continue except Exception, e: logging.error("Fatal Error: %s", e) - if self.session.debug: + if self.session.GetParameter("debug"): pdb.post_mortem() + raise ## A full iteration through all the classes without anyone diff --git a/rekall/plugins/guess_profile.py b/rekall/plugins/guess_profile.py index 4d69c24e9..15c579ca3 100644 --- a/rekall/plugins/guess_profile.py +++ b/rekall/plugins/guess_profile.py @@ -103,7 +103,8 @@ def DetectFromHit(self, hit, file_offset, address_space): config.DeclareOption("autodetect", group="Autodetection Overrides", - nargs="+", choices=utils.JITIterator(DetectionMethod), + type="ChoiceArray", + choices=utils.JITIterator(DetectionMethod), default=["nt_index", "osx", "pe", "rsds", "linux"], help="Autodetection method.") @@ -111,7 +112,7 @@ def DetectFromHit(self, hit, file_offset, address_space): group="Autodetection Overrides", help="Worst acceptable match for profile autodetection." + " (Default 1.0)", - type=float) + type="Float") class WindowsIndexDetector(DetectionMethod): diff --git a/rekall/plugins/imagecopy.py b/rekall/plugins/imagecopy.py index 0a4666c0b..f30bd05a4 100644 --- a/rekall/plugins/imagecopy.py +++ b/rekall/plugins/imagecopy.py @@ -76,7 +76,7 @@ def render(self, renderer): raise plugin.PluginError("Please provide an output-image filename") if (os.path.exists(self.output_image) and - os.path.getsize(self.output_image) > 1): + os.path.getsize(self.output_image) > 1): raise plugin.PluginError("Refusing to overwrite an existing file, " "please remove it before continuing") diff --git a/rekall/plugins/overlays/basic.py b/rekall/plugins/overlays/basic.py index 715bd1d50..9af5e1259 100644 --- a/rekall/plugins/overlays/basic.py +++ b/rekall/plugins/overlays/basic.py @@ -458,13 +458,13 @@ def find_all_lists(self, seen): if item not in seen: seen.append(item) - Blink = item.m(self._backward).dereference() + Blink = item.m(self._backward) if Blink.is_valid(): - stack.append(Blink) + stack.append(Blink.dereference()) - Flink = item.m(self._forward).dereference() + Flink = item.m(self._forward) if Flink.is_valid(): - stack.append(Flink) + stack.append(Flink.dereference()) def list_of_type(self, type, member): result = [] diff --git a/rekall/plugins/overlays/windows/common.py b/rekall/plugins/overlays/windows/common.py index 695f45a86..3dc35def2 100644 --- a/rekall/plugins/overlays/windows/common.py +++ b/rekall/plugins/overlays/windows/common.py @@ -29,6 +29,8 @@ from rekall import utils from rekall.plugins.overlays.windows import pe_vtypes +from rekall.plugins.overlays.windows import undocumented + MM_PROTECTION_ENUM = utils.EnumerationFromDefines(""" // @@ -112,7 +114,7 @@ 'IDT': lambda x: x.m("IDT") or x.m("IdtBase"), 'GDT': lambda x: x.m("GDT") or x.m("GdtBase"), 'KdVersionBlock': [None, ['Pointer', dict( - target='_KDDEBUGGER_DATA64')]], + target='_DBGKD_GET_VERSION64')]], }], '_KPRCB': [None, { @@ -440,7 +442,7 @@ '_DISPATCHER_HEADER': [None, { "Type": [None, ["Enumeration", dict( - enum_name="_KOBJECTS", + choices=undocumented.ENUMS["_KOBJECTS"], target="unsigned char", )]], }], @@ -661,6 +663,18 @@ def __unicode__(self): class _EPROCESS(obj.Struct): """ An extensive _EPROCESS with bells and whistles """ + def is_valid(self): + """Validate the _EPROCESS.""" + # PID must be in a reasonable range. + if self.pid < 0 or self.pid > 0xFFFF: + return False + + # Dispatch header must be for a process object. + if self.Pcb.Header.Type != "ProcessObject": + return False + + return True + @property def Peb(self): """ Returns a _PEB object which is using the process address space. @@ -1123,23 +1137,43 @@ class _EX_FAST_REF(obj.Struct): def __init__(self, target=None, **kwargs): self.target = target super(_EX_FAST_REF, self).__init__(**kwargs) + end_bit = self.RefCnt.end_bit + self.mask = ~ (2 ** end_bit - 1) + self._object = None + + def is_valid(self): + if self.Object.v() == 0: + return False + + return True + + @property + def Object(self): + if self._object is None: + result = self.m("Object") + self._object = result.cast(value=result.v() & self.mask) + + return self._object def dereference(self, vm=None): if self.target is None: raise AttributeError( "No target specified for dereferencing an _EX_FAST_REF.") - return self.dereference_as(self.target) + if not self.is_valid(): + return obj.NoneObject("_EX_FAST_REF not valid") - def dereference_as(self, type_name, parent=None, vm=None, **kwargs): + return self.Object.dereference_as(self.target) + + def dereference_as(self, type_name, parent=None, **kwargs): """Use the _EX_FAST_REF.Object pointer to resolve an object of the specified type. """ + if not self.is_valid(): + return obj.NoneObject("_EX_FAST_REF not valid") + parent = parent or self.obj_parent or self - MAX_FAST_REF = self.obj_profile.constants['MAX_FAST_REF'] - return self.obj_profile.Object( - type_name=type_name, offset=self.m("Object").v() & ~MAX_FAST_REF, - vm=vm or self.obj_vm, parent=parent, **kwargs) + return self.Object.dereference_as(type_name, parent=parent, **kwargs) def __getattr__(self, attr): return getattr(self.dereference(), attr) diff --git a/rekall/plugins/overlays/windows/pe_vtypes.py b/rekall/plugins/overlays/windows/pe_vtypes.py index 1d537b44e..4f6de70f3 100644 --- a/rekall/plugins/overlays/windows/pe_vtypes.py +++ b/rekall/plugins/overlays/windows/pe_vtypes.py @@ -888,7 +888,7 @@ def IAT(self): for directory in import_directory: dll = directory.Name.dereference() for thunk in directory.FirstThunk.dereference(): - function = thunk.u1.Function.cast("Pointer") + function = thunk.u1.Function yield dll, function, thunk.u1.Ordinal @@ -1078,4 +1078,3 @@ def read(self, addr, length): addr += len(data) return result - diff --git a/rekall/plugins/renderers/json_storage.py b/rekall/plugins/renderers/json_storage.py index 39e320614..d6cc8e517 100644 --- a/rekall/plugins/renderers/json_storage.py +++ b/rekall/plugins/renderers/json_storage.py @@ -74,7 +74,7 @@ class FileAddressSpaceObjectRenderer( - json_renderer.BaseAddressSpaceObjectRenderer): + json_renderer.BaseAddressSpaceObjectRenderer): renders_type = "FileAddressSpace" def GetState(self, item, **options): @@ -85,10 +85,6 @@ def GetState(self, item, **options): return state -class EWFAddressSpaceObjectRenderer(FileAddressSpaceObjectRenderer): - renders_type = "EWFAddressSpace" - - class AttributeDictObjectRenderer(json_renderer.StateBasedObjectRenderer): renders_type = "AttributeDict" @@ -103,7 +99,7 @@ def DecodeFromJsonSafe(self, state, options): class IA32PagedMemoryObjectRenderer( - json_renderer.BaseAddressSpaceObjectRenderer): + json_renderer.BaseAddressSpaceObjectRenderer): renders_type = "IA32PagedMemory" def GetState(self, item, **options): @@ -114,7 +110,7 @@ def GetState(self, item, **options): return state class MIPSPagedMemoryObjectRenderer( - json_renderer.BaseAddressSpaceObjectRenderer): + json_renderer.BaseAddressSpaceObjectRenderer): renders_type = "MipsAddressSpace" def GetState(self, item, **options): diff --git a/rekall/plugins/renderers/windows.py b/rekall/plugins/renderers/windows.py index 359bb2c77..28a172e70 100644 --- a/rekall/plugins/renderers/windows.py +++ b/rekall/plugins/renderers/windows.py @@ -45,7 +45,8 @@ def EncodeToJsonSafe(self, task, **_): ) ) - return json_renderer.JsonObjectRenderer.EncodeToJsonSafe(self, result) + res = json_renderer.JsonObjectRenderer.EncodeToJsonSafe(self, result) + return res def Summary(self, item, **_): return "%s (%s)" % (item.get("Cybox", {}).get("Name", ""), diff --git a/rekall/plugins/tools/json_tools.py b/rekall/plugins/tools/json_tools.py index 0fb267376..b5cd2b461 100644 --- a/rekall/plugins/tools/json_tools.py +++ b/rekall/plugins/tools/json_tools.py @@ -143,58 +143,3 @@ def BuildBaselineData(self, config_options): baseline = super(TestJSONParser, self).BuildBaselineData(config_options) return baseline - - -class Sampler(object): - handlers = {} - - def __init__(self, f): - self.f = f - Sampler.handlers[f.__name__] = f - - def __call__(self): - self.f() - - -class RenderingSampler(plugin.Command): - name = "render_sample" - - @classmethod - def args(cls, parser): - super(RenderingSampler, cls).args(parser) - - parser.add_argument("sample", choices=Sampler.handlers.keys(), - help="Sample to render.") - - @Sampler - def Nothing(self, renderer): - _ = renderer - - @Sampler - def Format(self, renderer): - renderer.format("This is a formatted string: %s %d %s", "foo", 42, - "bar") - - @Sampler - def UnnamedSection(self, renderer): - renderer.section() - - @Sampler - def NamedSection(self, renderer): - renderer.section("Named Section") - - @Sampler - def OneRowTable(self, renderer): - renderer.table_header([('Parameter', 'parameter', '30'), - (' Documentation', 'doc', '70')]) - renderer.table_row("important-parameter", 42) - - def __init__(self, sample=None, **kwargs): - super(RenderingSampler, self).__init__(**kwargs) - - if sample is None: - raise ValueError("sample argument can't be None") - self.sample = sample - - def render(self, renderer): - Sampler.handlers[self.sample](self, renderer) diff --git a/rekall/plugins/tools/profile_tool.py b/rekall/plugins/tools/profile_tool.py index f9d60c0b8..a1bd7a58e 100755 --- a/rekall/plugins/tools/profile_tool.py +++ b/rekall/plugins/tools/profile_tool.py @@ -411,6 +411,10 @@ class TestConvertProfile(testlib.DisabledTest): PARAMETERS = dict(commandline="convert_profile") +class TestBuildIndex(testlib.DisabledTest): + PARAMETERS = dict(commandline="build_index") + + class BuildIndex(plugin.Command): """Generate a profile index file based on an index specification. diff --git a/rekall/plugins/windows/address_resolver.py b/rekall/plugins/windows/address_resolver.py index 5b22095d0..dbee119fc 100644 --- a/rekall/plugins/windows/address_resolver.py +++ b/rekall/plugins/windows/address_resolver.py @@ -28,6 +28,7 @@ from rekall.plugins.common import address_resolver from rekall.plugins.windows import common from rekall.plugins.overlays.windows import pe_vtypes +from rekall.plugins.overlays.windows import windows class KernelModule(object): @@ -188,11 +189,12 @@ def LoadProfileForDll(self, module_base, module_name): "default_address_space")) constants = {} - for _, func, name, _ in peinfo.pe_helper.ExportDirectory(): - self.session.report_progress("Merging export table: %s", name) - func_offset = func.v() - if not result.get_constant_by_address(func_offset): - constants[str(name or "")] = func_offset - module_base + if "Export" in self.session.GetParameter("name_resolution_strategies"): + for _, func, name, _ in peinfo.pe_helper.ExportDirectory(): + self.session.report_progress("Merging export table: %s", name) + func_offset = func.v() + if not result.get_constant_by_address(func_offset): + constants[str(name or "")] = func_offset - module_base result.add_constants(constants_are_addresses=True, **constants) @@ -225,13 +227,14 @@ def LoadProfileForModule(self, module): address_space=module.obj_vm) constants = {} - for _, func, name, _ in peinfo.pe_helper.ExportDirectory(): - self.session.report_progress("Merging export table: %s", name) - func_offset = func.v() - if not result.get_constant_by_address(func_offset): - constants[str(name or "")] = func_offset - module_base + if "Export" in self.session.GetParameter("name_resolution_strategies"): + for _, func, name, _ in peinfo.pe_helper.ExportDirectory(): + self.session.report_progress("Merging export table: %s", name) + func_offset = func.v() + if not result.get_constant_by_address(func_offset): + constants[str(name or "")] = func_offset - module_base - result.add_constants(constants_are_addresses=True, **constants) + result.add_constants(constants_are_addresses=True, **constants) self.profiles[module_name] = result @@ -470,18 +473,22 @@ def _EnsureInitialized(self): self.modules_by_name[module.name] = module self.section_map.insert((module.base, module)) - # Extract all exported symbols into the profile's symbol table. - for _, func, name, _ in self.pe_helper.ExportDirectory(): - func_address = func.v() - try: - symbols[utils.SmartUnicode(name)] = func_address - except ValueError: - continue - - # Load the profile for this binary. - self.pe_profile = self.session.LoadProfile("%s/GUID/%s" % ( - utils.SmartUnicode(self.pe_helper.RSDS.Filename).split(".")[0], - self.pe_helper.RSDS.GUID_AGE)) + if "Export" in self.session.GetParameter("name_resolution_strategies"): + # Extract all exported symbols into the profile's symbol table. + for _, func, name, _ in self.pe_helper.ExportDirectory(): + func_address = func.v() + try: + symbols[utils.SmartUnicode(name)] = func_address + except ValueError: + continue + + if "Symbol" in self.session.GetParameter("name_resolution_strategies"): + # Load the profile for this binary. + self.pe_profile = self.session.LoadProfile("%s/GUID/%s" % ( + utils.SmartUnicode(self.pe_helper.RSDS.Filename).split(".")[0], + self.pe_helper.RSDS.GUID_AGE)) + else: + self.pe_profile = windows.BasicPEProfile(session=self.session) self.pe_profile.image_base = self.image_base diff --git a/rekall/plugins/windows/disassembler.py b/rekall/plugins/windows/disassembler.py index 37b17ed6c..8597761b0 100644 --- a/rekall/plugins/windows/disassembler.py +++ b/rekall/plugins/windows/disassembler.py @@ -361,7 +361,10 @@ def render(self, renderer): class TestDisassemble(testlib.SimpleTestCase): PARAMETERS = dict( - commandline="dis -l %(length)s %(func)s", + # We want to test symbol discovery via export table detection so turn it + # on. + commandline=("dis -l %(length)s %(func)s " + "--name_resolution_strategies Export"), func=0x805031be, length=20 ) diff --git a/rekall/plugins/windows/gui/autodetect.py b/rekall/plugins/windows/gui/autodetect.py index 10111582c..74fbb1341 100644 --- a/rekall/plugins/windows/gui/autodetect.py +++ b/rekall/plugins/windows/gui/autodetect.py @@ -58,7 +58,7 @@ def GetWin32kOverlay(self, win32k_profile): overlay = dict(tagDESKTOP=[None, {}], tagWINDOWSTATION=[None, {}], tagTHREADINFO=[None, {}], - ) + ) with self.session.plugins.cc() as cc: for task in self.session.plugins.pslist().filter_processes(): @@ -115,30 +115,30 @@ def Get_tagWINDOWSTATION_overlay(self, overlay): logging.debug("Checking tagWINDOWSTATION at %#x", offset) for o, info in self.analyze_struct.GuessMembers(offset, size=0x200): if self._AddField( - "Tag:Win", info, "rpwinstaNext", fields, - [o, ["Pointer", dict( - target="tagWINDOWSTATION" + "Tag:Win", info, "rpwinstaNext", fields, + [o, ["Pointer", dict( + target="tagWINDOWSTATION" )]]): continue elif self._AddField( - "Tag:Des", info, "rpdeskList", fields, - [o, ["Pointer", dict( - target="tagDESKTOP" + "Tag:Des", info, "rpdeskList", fields, + [o, ["Pointer", dict( + target="tagDESKTOP" )]]): continue elif self._AddField( - "Tag:AtmT", info, "pGlobalAtomTable", fields, - [o, ["Pointer", dict( - target="_RTL_ATOM_TABLE" + "Tag:AtmT", info, "pGlobalAtomTable", fields, + [o, ["Pointer", dict( + target="_RTL_ATOM_TABLE" )]]): continue elif self._AddField( - "Const:win32k!gTerm", info, "pTerm", fields, - [o, ["Pointer", dict( - target="tagTERMINAL" + "Const:win32k!gTerm", info, "pTerm", fields, + [o, ["Pointer", dict( + target="tagTERMINAL" )]]): continue @@ -177,7 +177,7 @@ def Get_tagDESKTOP_overlay(self, overlay): desktops.add(offset) for o, info in self.analyze_struct.GuessMembers( - offset, search=0x400): + offset, search=0x400): if self._AddField("Tag:Des", info, "rpdeskNext", fields, [o, ["Pointer", dict( @@ -251,7 +251,7 @@ def _Check_tagPROCESSINFO(self, offset): def _AnalyzeTagTHREADINFO(self, offset, fields): logging.debug("Checking tagTHREADINFO at %#x", offset) for o, info in self.analyze_struct.GuessMembers( - offset, size=0x400, search=0x600): + offset, size=0x400, search=0x600): if self._AddField("Tag:Thr", info, "pEThread", fields, [o, ["Pointer", dict( @@ -302,7 +302,7 @@ def Get_tagTHREADINFO_overlay(self, overlay): # Iterate over all tagTHREADINFO objects. thread_infos = set() for wndstation in self.wndstation().rpwinstaNext.walk_list( - "rpwinstaNext"): + "rpwinstaNext"): for desktop in wndstation.rpdeskList.walk_list("rpdeskNext"): thread_info_pool = self.analyze_struct.SearchForPoolHeader( desktop.PtiList.Flink.v(), search=0x600) diff --git a/rekall/plugins/windows/gui/userhandles.py b/rekall/plugins/windows/gui/userhandles.py index a71d488fe..75cb93168 100644 --- a/rekall/plugins/windows/gui/userhandles.py +++ b/rekall/plugins/windows/gui/userhandles.py @@ -88,7 +88,7 @@ def handles(self): # Skip pids that do not match. if (self.filtering_requested and - handle.Process.UniqueProcessId not in pids): + handle.Process.UniqueProcessId not in pids): continue # Allow the user to match of handle type. @@ -128,7 +128,7 @@ def render(self, renderer): ("Thread", "thread", "^8"), ("Process", "process", "5"), ("Process Name", "process_name", ""), - ]) + ]) renderer.table_row( handle, @@ -215,7 +215,7 @@ def render(self, renderer): ("Tag", "tag", "8"), ("fnDestroy", "fnDestroy", "[addrpad]"), ("Flags", "flags", ""), - ]) + ]) for session in self.session.plugins.sessions().session_spaces(): for handle in self.gahti(session): @@ -317,7 +317,7 @@ def render(self, renderer): ("Flags", "flags", "10"), ("Function", "function", "[addrpad]"), ("Module", "module", ""), - ]) + ]) atoms_plugin = self.session.plugins.atoms() for session in self.session.plugins.sessions().session_spaces(): diff --git a/rekall/plugins/windows/gui/win32k_core.py b/rekall/plugins/windows/gui/win32k_core.py index f679dbdaa..a0a861400 100644 --- a/rekall/plugins/windows/gui/win32k_core.py +++ b/rekall/plugins/windows/gui/win32k_core.py @@ -395,8 +395,9 @@ def ThreadOwned(self): def ProcessOwned(self): """Handles of these types are always process owned""" return str(self.bType) in [ - 'TYPE_MENU', 'TYPE_CURSOR', 'TYPE_TIMER', - 'TYPE_CALLPROC', 'TYPE_ACCELTABLE'] + 'TYPE_MENU', 'TYPE_CURSOR', 'TYPE_TIMER', + 'TYPE_CALLPROC', 'TYPE_ACCELTABLE'] + @property def Thread(self): """Return the ETHREAD if its thread owned""" diff --git a/rekall/plugins/windows/handles.py b/rekall/plugins/windows/handles.py index e726a1bc1..d24deab1b 100644 --- a/rekall/plugins/windows/handles.py +++ b/rekall/plugins/windows/handles.py @@ -114,4 +114,3 @@ def render(self, renderer): handle.HandleValue, handle.GrantedAccess, object_type, name) - diff --git a/rekall/plugins/windows/interactive/structs.py b/rekall/plugins/windows/interactive/structs.py index 02fa30bae..e72307f5e 100644 --- a/rekall/plugins/windows/interactive/structs.py +++ b/rekall/plugins/windows/interactive/structs.py @@ -144,6 +144,7 @@ def render(self, renderer): if pool_header: name = (pool_header.ProcessBilled.name or str(pool_header.Tag).encode("string-escape")) + renderer.format( "{0:#x} is inside pool allocation with tag '{1}' ({2:#x})\n", self.offset, name, pool_header) diff --git a/rekall/plugins/windows/kernel.py b/rekall/plugins/windows/kernel.py index 637c2bdac..27a398a92 100644 --- a/rekall/plugins/windows/kernel.py +++ b/rekall/plugins/windows/kernel.py @@ -60,8 +60,9 @@ def calculate(self): else: kernel_boundary = 0x80000000 + maxlen = 0xFFFFF87FFFFFFFFF - kernel_boundary kernel_boundary = obj.Pointer.integer_to_address(kernel_boundary) - for hit in scanner.scan(offset=kernel_boundary, maxlen=2**64): + for hit in scanner.scan(offset=kernel_boundary, maxlen=maxlen): # Search backwards for an MZ signature on the page boundary. page = hit & 0xFFFFFFFFFFFFF000 diff --git a/rekall/plugins/windows/kpcr.py b/rekall/plugins/windows/kpcr.py index 690c9df9c..38224ea60 100644 --- a/rekall/plugins/windows/kpcr.py +++ b/rekall/plugins/windows/kpcr.py @@ -76,7 +76,7 @@ def render(self, renderer): current_thread, current_thread.Cid.UniqueThread, current_thread.owning_process().ImageFileName, current_thread.Cid.UniqueProcess, - ) + ) if idle_thread: renderer.format("{0:<30}: {1:#x} TID {2} ({3}:{4})\n", @@ -84,7 +84,7 @@ def render(self, renderer): idle_thread, idle_thread.Cid.UniqueThread, idle_thread.owning_process().ImageFileName, idle_thread.Cid.UniqueProcess, - ) + ) if next_thread: renderer.format("{0:<30}: {1:#x} TID {2} ({3}:{4})\n", @@ -93,7 +93,7 @@ def render(self, renderer): next_thread.Cid.UniqueThread, next_thread.owning_process().ImageFileName, next_thread.Cid.UniqueProcess, - ) + ) renderer.format("{0:<30}: CPU {1} ({2} @ {3} MHz)\n", "Details", diff --git a/rekall/plugins/windows/malware/apihooks.py b/rekall/plugins/windows/malware/apihooks.py index f59e6a6f3..a62bfc346 100644 --- a/rekall/plugins/windows/malware/apihooks.py +++ b/rekall/plugins/windows/malware/apihooks.py @@ -278,9 +278,17 @@ def detect_IAT_hooks(self): resolver = self.session.address_resolver for idx, (dll, func_address, _) in enumerate(pe.IAT()): - target_dll, target_func_name = imports[idx] - target_dll = self.session.address_resolver.NormalizeModuleName( - target_dll) + + try: + target_dll, target_func_name = imports[idx] + target_dll = self.session.address_resolver.NormalizeModuleName( + target_dll) + except IndexError: + # We can not retrieve these function's name from the + # OriginalFirstThunk array - possibly because it is not mapped + # in. + target_dll = dll + target_func_name = "" self.session.report_progress( "Checking function %s!%s", target_dll, target_func_name) diff --git a/rekall/plugins/windows/malware/callbacks.py b/rekall/plugins/windows/malware/callbacks.py index d88804e7c..1be0fe128 100644 --- a/rekall/plugins/windows/malware/callbacks.py +++ b/rekall/plugins/windows/malware/callbacks.py @@ -26,61 +26,61 @@ callback_types = { '_NOTIFICATION_PACKET' : [0x10, { - 'ListEntry' : [0x0, ['_LIST_ENTRY']], - 'DriverObject' : [0x8, ['pointer', ['_DRIVER_OBJECT']]], - 'NotificationRoutine' : [0xC, ['unsigned int']], - }], + 'ListEntry' : [0x0, ['_LIST_ENTRY']], + 'DriverObject' : [0x8, ['pointer', ['_DRIVER_OBJECT']]], + 'NotificationRoutine' : [0xC, ['unsigned int']], + }], '_KBUGCHECK_CALLBACK_RECORD' : [0x20, { - 'Entry' : [0x0, ['_LIST_ENTRY']], - 'CallbackRoutine' : [0x8, ['unsigned int']], - 'Buffer' : [0xC, ['pointer', ['void']]], - 'Length' : [0x10, ['unsigned int']], - 'Component' : [0x14, ['pointer', ['String', dict(length=64)]]], - 'Checksum' : [0x18, ['pointer', ['unsigned int']]], - 'State' : [0x1C, ['unsigned char']], - }], + 'Entry' : [0x0, ['_LIST_ENTRY']], + 'CallbackRoutine' : [0x8, ['unsigned int']], + 'Buffer' : [0xC, ['pointer', ['void']]], + 'Length' : [0x10, ['unsigned int']], + 'Component' : [0x14, ['pointer', ['String', dict(length=64)]]], + 'Checksum' : [0x18, ['pointer', ['unsigned int']]], + 'State' : [0x1C, ['unsigned char']], + }], '_KBUGCHECK_REASON_CALLBACK_RECORD' : [0x1C, { - 'Entry' : [0x0, ['_LIST_ENTRY']], - 'CallbackRoutine' : [0x8, ['unsigned int']], - 'Component' : [0xC, ['pointer', ['String', dict(length=8)]]], - 'Checksum' : [0x10, ['pointer', ['unsigned int']]], - 'Reason' : [0x14, ['unsigned int']], - 'State' : [0x18, ['unsigned char']], - }], + 'Entry' : [0x0, ['_LIST_ENTRY']], + 'CallbackRoutine' : [0x8, ['unsigned int']], + 'Component' : [0xC, ['pointer', ['String', dict(length=8)]]], + 'Checksum' : [0x10, ['pointer', ['unsigned int']]], + 'Reason' : [0x14, ['unsigned int']], + 'State' : [0x18, ['unsigned char']], + }], '_SHUTDOWN_PACKET' : [0xC, { - 'Entry' : [0x0, ['_LIST_ENTRY']], - 'DeviceObject' : [0x8, ['pointer', ['_DEVICE_OBJECT']]], - }], + 'Entry' : [0x0, ['_LIST_ENTRY']], + 'DeviceObject' : [0x8, ['pointer', ['_DEVICE_OBJECT']]], + }], '_EX_CALLBACK_ROUTINE_BLOCK' : [0x8, { - 'RundownProtect' : [0x0, ['unsigned int']], - 'Function' : [0x4, ['unsigned int']], - 'Context' : [0x8, ['unsigned int']], - }], + 'RundownProtect' : [0x0, ['unsigned int']], + 'Function' : [0x4, ['unsigned int']], + 'Context' : [0x8, ['unsigned int']], + }], '_GENERIC_CALLBACK' : [0xC, { - 'Callback' : [0x4, ['pointer', ['void']]], - 'Associated' : [0x8, ['pointer', ['void']]], - }], + 'Callback' : [0x4, ['pointer', ['void']]], + 'Associated' : [0x8, ['pointer', ['void']]], + }], '_REGISTRY_CALLBACK_LEGACY' : [0x38, { - 'CreateTime' : [0x0, ['WinFileTime', {}]], - }], + 'CreateTime' : [0x0, ['WinFileTime', {}]], + }], '_REGISTRY_CALLBACK' : [None, { - 'ListEntry' : [0x0, ['_LIST_ENTRY']], - 'Function' : [0x1C, ['pointer', ['void']]], - }], + 'ListEntry' : [0x0, ['_LIST_ENTRY']], + 'Function' : [0x1C, ['pointer', ['void']]], + }], '_DBGPRINT_CALLBACK' : [0x14, { - 'Function' : [0x8, ['pointer', ['void']]], - }], + 'Function' : [0x8, ['pointer', ['void']]], + }], '_NOTIFY_ENTRY_HEADER' : [None, { - 'ListEntry' : [0x0, ['_LIST_ENTRY']], - 'EventCategory' : [0x8, ['Enumeration', dict( - target='long', choices={ - 0: 'EventCategoryReserved', - 1: 'EventCategoryHardwareProfileChange', - 2: 'EventCategoryDeviceInterfaceChange', - 3: 'EventCategoryTargetDeviceChange'})]], - 'CallbackRoutine' : [0x14, ['unsigned int']], - 'DriverObject' : [0x1C, ['pointer', ['_DRIVER_OBJECT']]], - }], + 'ListEntry' : [0x0, ['_LIST_ENTRY']], + 'EventCategory' : [0x8, ['Enumeration', dict( + target='long', choices={ + 0: 'EventCategoryReserved', + 1: 'EventCategoryHardwareProfileChange', + 2: 'EventCategoryDeviceInterfaceChange', + 3: 'EventCategoryTargetDeviceChange'})]], + 'CallbackRoutine' : [0x14, ['unsigned int']], + 'DriverObject' : [0x1C, ['pointer', ['_DRIVER_OBJECT']]], + }], } @@ -94,8 +94,8 @@ def sanity_check(self, vm): """ if (not vm.is_valid_address(self.Entry.Flink) or - not vm.is_valid_address(self.Entry.Blink) or - not vm.is_valid_address(self.DeviceObject)): + not vm.is_valid_address(self.Entry.Blink) or + not vm.is_valid_address(self.DeviceObject)): return False # Dereference the device object @@ -120,7 +120,7 @@ class PoolScanFSCallback(AbstractCallbackScanner): ('CheckPoolSize', dict(condition=lambda x: x == 0x18)), ('CheckPoolType', dict(non_paged=True, paged=True, free=True)), - ] + ] def scan(self, **kwargs): for pool_header in super(PoolScanFSCallback, self).scan(**kwargs): @@ -139,7 +139,7 @@ class PoolScanShutdownCallback(AbstractCallbackScanner): ('CheckPoolType', dict(non_paged=True, paged=True, free=True)), ('CheckPoolIndex', dict(value=0)), - ] + ] def __init__(self, kernel_address_space=None, **kwargs): super(PoolScanShutdownCallback, self).__init__(**kwargs) @@ -148,7 +148,7 @@ def __init__(self, kernel_address_space=None, **kwargs): def scan(self, offset=0, **kwargs): for pool_header in super(PoolScanShutdownCallback, self).scan( - offset=offset, **kwargs): + offset=offset, **kwargs): # Instantiate the object in physical space but give it a native VM # of kernel space @@ -174,7 +174,7 @@ class PoolScanGenericCallback(AbstractCallbackScanner): checks = [('PoolTagCheck', dict(tag="Cbrb")), ('CheckPoolSize', dict(condition=lambda x: x == 0x18)), ('CheckPoolType', dict(non_paged=True, paged=True, free=True)), - ] + ] def scan(self, **kwargs): """ @@ -207,7 +207,7 @@ class PoolScanDbgPrintCallback(AbstractCallbackScanner): def scan(self, offset=0, **kwargs): """Enumerate DebugPrint callbacks on Vista and 7""" for pool_header in super(PoolScanDbgPrintCallback, self).scan( - offset=offset, **kwargs): + offset=offset, **kwargs): callback = self.profile.Object( '_DBGPRINT_CALLBACK', offset=pool_header.end(), @@ -223,7 +223,7 @@ class PoolScanRegistryCallback(AbstractCallbackScanner): ('CheckPoolSize', dict(condition=lambda x: x >= 0x38)), ('CheckPoolType', dict(non_paged=True, paged=True, free=True)), ('CheckPoolIndex', dict(value=4)), - ] + ] def scan(self, offset=0, **kwargs): """ @@ -233,7 +233,7 @@ def scan(self, offset=0, **kwargs): or CmRegisterCallbackEx. """ for pool_header in super(PoolScanRegistryCallback, self).scan( - offset=offset, **kwargs): + offset=offset, **kwargs): callback = self.profile.Object( '_REGISTRY_CALLBACK', offset=pool_header.end(), @@ -249,7 +249,7 @@ class PoolScanPnp9(AbstractCallbackScanner): ('CheckPoolSize', dict(condition=lambda x: x >= 0x30)), ('CheckPoolType', dict(non_paged=True, paged=True, free=True)), ('CheckPoolIndex', dict(value=1)), - ] + ] def __init__(self, kernel_address_space=None, **kwargs): self.kernel_address_space = kernel_address_space @@ -258,7 +258,7 @@ def __init__(self, kernel_address_space=None, **kwargs): def scan(self, offset=0, **kwargs): """Enumerate IoRegisterPlugPlayNotification""" for pool_header in super(PoolScanPnp9, self).scan( - offset=offset, **kwargs): + offset=offset, **kwargs): entry = self.profile.Object( "_NOTIFY_ENTRY_HEADER", offset=pool_header.end(), vm=self.address_space) @@ -272,7 +272,7 @@ def scan(self, offset=0, **kwargs): "_OBJECT_HEADER", offset=(driver.obj_offset - driver.obj_profile.get_obj_offset( - "_OBJECT_HEADER", "Body")), + "_OBJECT_HEADER", "Body")), vm=driver.obj_vm) # Grab the object name @@ -301,8 +301,8 @@ def __init__(self, scan_in_kernel_address_space=False, **kwargs): # Add some plugin specific vtypes. self.profile.add_types(callback_types) self.profile.add_classes({ - '_SHUTDOWN_PACKET': _SHUTDOWN_PACKET, - }) + '_SHUTDOWN_PACKET': _SHUTDOWN_PACKET, + }) self.profile = self.profile.copy() pe_vtypes.PEProfile.Initialize(self.profile) @@ -326,17 +326,20 @@ def get_kernel_callbacks(self): for symbol in routines: # The list is an array of 8 _EX_FAST_REF objects - addrs = self.profile.get_constant_object( + callbacks = self.profile.get_constant_object( symbol, target="Array", target_args=dict( count=8, - target='_EX_FAST_REF') + target='_EX_FAST_REF', + target_args=dict( + target="_GENERIC_CALLBACK", + ) ) + ) - for addr in addrs: - callback = addr.dereference_as("_GENERIC_CALLBACK") - if callback: + for callback in callbacks: + if callback.Callback: yield "GenericKernelCallback", callback.Callback, None def get_bugcheck_callbacks(self): @@ -351,7 +354,7 @@ def get_bugcheck_callbacks(self): target='_LIST_ENTRY')) for l in KeBugCheckCallbackListHead.list_of_type( - "_KBUGCHECK_CALLBACK_RECORD", "Entry"): + "_KBUGCHECK_CALLBACK_RECORD", "Entry"): yield ("KeBugCheckCallbackListHead", l.CallbackRoutine, l.Component.dereference()) @@ -387,7 +390,7 @@ def get_bugcheck_reason_callbacks(self): target="_LIST_ENTRY") for l in bugs.list_of_type( - "_KBUGCHECK_REASON_CALLBACK_RECORD", "Entry"): + "_KBUGCHECK_REASON_CALLBACK_RECORD", "Entry"): yield ("KeRegisterBugCheckReasonCallback", l.CallbackRoutine, l.Component.dereference()) @@ -458,7 +461,7 @@ def render(self, renderer): ("Callback", "callback", "[addrpad]"), ("Module", "module", "20"), ("Details", "details", ""), - ]) + ]) # We use the modules plugin to help locate the module containing the diff --git a/rekall/plugins/windows/modules.py b/rekall/plugins/windows/modules.py index 0fab90b91..b8cb8d53a 100644 --- a/rekall/plugins/windows/modules.py +++ b/rekall/plugins/windows/modules.py @@ -179,7 +179,7 @@ def args(cls, parser): parser.add_argument("--name_regex", help="Filter module names by this regex.") - parser.add_argument("scan-filename", required=False, + parser.add_argument("scan_filename", required=False, help="Optional file to scan. If not specified " "we scan the physical address space.") diff --git a/rekall/plugins/windows/pagefile.py b/rekall/plugins/windows/pagefile.py index 2d518069b..57bc02e5b 100644 --- a/rekall/plugins/windows/pagefile.py +++ b/rekall/plugins/windows/pagefile.py @@ -276,22 +276,34 @@ def vtop(self, vaddr): The function should return either None (no valid mapping) or the offset in physical memory where the address maps. ''' - pde_value = self.get_pde(vaddr) - if not self.entry_present(pde_value): - # If PDE is not valid the page table does not exist yet. According - # to - # http://i-web.i.u-tokyo.ac.jp/edu/training/ss/lecture/new-documents/Lectures/14-AdvVirtualMemory/AdvVirtualMemory.pdf - # slide 11 this is the same as PTE of zero - i.e. consult the VAD. - return self.get_phys_addr(vaddr, 0) - - if self.page_size_flag(pde_value): - return self.get_four_meg_paddr(vaddr, pde_value) - - pte_value = self.get_pte(vaddr, pde_value) - if not self.entry_present(pte_value): - return None + try: + return self._tlb.Get(vaddr) + except KeyError: + pdpte = self.get_pdpte(vaddr) + if not self.entry_present(pdpte): + return None + + pde = self.get_pde(vaddr, pdpte) + if not self.entry_present(pde): + # If PDE is not valid the page table does not exist + # yet. According to + # http://i-web.i.u-tokyo.ac.jp/edu/training/ss/lecture/new-documents/Lectures/14-AdvVirtualMemory/AdvVirtualMemory.pdf + # slide 11 this is the same as PTE of zero - i.e. consult the + # VAD. + if not self._resolve_vads: + return None + + return self.get_phys_addr(vaddr, 0) - return self.get_phys_addr(vaddr, pte_value) + if self.page_size_flag(pde): + return self.get_two_meg_paddr(vaddr, pde) + + pte = self.get_pte(vaddr, pde) + + res = self.get_phys_addr(vaddr, pte) + + self._tlb.Put(vaddr, res) + return res class WindowsAMD64PagedMemory(WindowsPagedMemoryMixin, amd64.AMD64PagedMemory): @@ -307,36 +319,44 @@ def vtop(self, vaddr): The function returns either None (no valid mapping) or the offset in physical memory where the address maps. ''' - vaddr = long(vaddr) - pml4e = self.get_pml4e(vaddr) - if not self.entry_present(pml4e): - # Add support for paged out PML4E - return None - - pdpte = self.get_pdpte(vaddr, pml4e) - if not self.entry_present(pdpte): - # Add support for paged out PDPTE - # Insert buffalo here! - return None - - if self.page_size_flag(pdpte): - return self.get_one_gig_paddr(vaddr, pdpte) - - pde = self.get_pde(vaddr, pdpte) - if not self.entry_present(pde): - # If PDE is not valid the page table does not exist yet. According - # to - # http://i-web.i.u-tokyo.ac.jp/edu/training/ss/lecture/new-documents/Lectures/14-AdvVirtualMemory/AdvVirtualMemory.pdf - # slide 11 this is the same PTE of zero. - return self.get_phys_addr(vaddr, 0) - - # Is this a 2 meg page? - if self.page_size_flag(pde): - return self.get_two_meg_paddr(vaddr, pde) - - pte = self.get_pte(vaddr, pde) - - return self.get_phys_addr(vaddr, pte) + try: + return self._tlb.Get(vaddr) + except KeyError: + vaddr = long(vaddr) + pml4e = self.get_pml4e(vaddr) + if not self.entry_present(pml4e): + # Add support for paged out PML4E + return None + + pdpte = self.get_pdpte(vaddr, pml4e) + if not self.entry_present(pdpte): + # Add support for paged out PDPTE + # Insert buffalo here! + return None + + if self.page_size_flag(pdpte): + return self.get_one_gig_paddr(vaddr, pdpte) + + pde = self.get_pde(vaddr, pdpte) + if not self.entry_present(pde): + # If PDE is not valid the page table does not exist + # yet. According to + # http://i-web.i.u-tokyo.ac.jp/edu/training/ss/lecture/new-documents/Lectures/14-AdvVirtualMemory/AdvVirtualMemory.pdf + # slide 11 this is the same PTE of zero. + if not self._resolve_vads: + return None + + return self.get_phys_addr(vaddr, 0) + + # Is this a 2 meg page? + if self.page_size_flag(pde): + return self.get_two_meg_paddr(vaddr, pde) + + pte = self.get_pte(vaddr, pde) + res = self.get_phys_addr(vaddr, pte) + + self._tlb.Put(vaddr, res) + return res class Pagefiles(common.WindowsCommandPlugin): diff --git a/rekall/plugins/windows/pas2kas.py b/rekall/plugins/windows/pas2kas.py index 604e2fcaf..0a168a270 100644 --- a/rekall/plugins/windows/pas2kas.py +++ b/rekall/plugins/windows/pas2kas.py @@ -139,7 +139,7 @@ def _get_virtual_address(self, physical_address, pid): bisect.bisect(lookup_map, [physical_address, 2**64, 0, 0])-1] if (lookup_pa <= physical_address and - lookup_pa + length > physical_address): + lookup_pa + length > physical_address): # Yield the pid and the virtual offset return lookup_va + physical_address - lookup_pa, task return None, None @@ -154,7 +154,7 @@ def render(self, renderer): for physical_address in self.physical_address: for virtual_address, task in self.get_virtual_address( - physical_address): + physical_address): if task is 'Kernel': renderer.table_row(physical_address, virtual_address, 0, 'Kernel') @@ -168,4 +168,4 @@ def render(self, renderer): class TestPas2Vas(testlib.SimpleTestCase): PARAMETERS = dict( commandline="pas2vas %(offset)s --pid 0 " - ) + ) diff --git a/rekall/plugins/windows/pfn.py b/rekall/plugins/windows/pfn.py index 8b726f0bc..33f07ff0f 100644 --- a/rekall/plugins/windows/pfn.py +++ b/rekall/plugins/windows/pfn.py @@ -87,7 +87,7 @@ def modify(cls, profile): }) -class VtoP(plugin.KernelASMixin, plugin.PhysicalASMixin, plugin.ProfileCommand): +class VtoP(common.WinProcessFilter): """Prints information about the virtual to physical translation.""" __name = "vtop" @@ -97,13 +97,10 @@ class VtoP(plugin.KernelASMixin, plugin.PhysicalASMixin, plugin.ProfileCommand): @classmethod def args(cls, parser): super(VtoP, cls).args(parser) - parser.add_argument("virtual_address", type="IntParser", + parser.add_argument("virtual_address", type="ArrayIntParser", help="The Virtual Address to examine.") - parser.add_argument("-a", "--address_space", default=None, - help="The address space to use.") - - def __init__(self, virtual_address=None, address_space=None, **kwargs): + def __init__(self, virtual_address=(), **kwargs): """Prints information about the virtual to physical translation. This is similar to windbg's !vtop extension. @@ -114,10 +111,11 @@ def __init__(self, virtual_address=None, address_space=None, **kwargs): kernel_address_space). """ super(VtoP, self).__init__(**kwargs) - load_as = self.session.plugins.load_as(session=self.session) - self.address_space = load_as.ResolveAddressSpace(address_space) + if not isinstance(virtual_address, (tuple, list)): + virtual_address = [virtual_address] - self.address = virtual_address + self.addresses = [self.session.address_resolver.get_address_by_name(x) + for x in virtual_address] def _vtop_32bit(self, vaddr, address_space): """An implementation specific to the 32 bit intel address space.""" @@ -238,34 +236,46 @@ def vtop(self, virtual_address, address_space=None): return function(virtual_address, address_space) - def render_pte(self, address, value, renderer): + def render_pte(self, address, value, renderer, vaddr): """Analyze the PTE in detail. This follows the algorithm in WindowsAMD64PagedMemory.get_phys_addr(). """ - pte_plugin = self.session.plugins.pte(address, "P", - self.address) + pte_plugin = self.session.plugins.pte(address, "P", vaddr) pte_plugin.render(renderer) pte = self.profile._MMPTE() pte.u.Long = value - phys_addr = self.address_space.ResolveProtoPTE( - pte, self.address) + phys_addr = self.address_space.ResolveProtoPTE(pte, vaddr) if phys_addr: renderer.format("PTE mapped at 0x{0:08X}\n", phys_addr) else: renderer.format("Invalid PTE\n") def render(self, renderer): - if self.address is None: - return + if self.filtering_requested: + with self.session.plugins.cc() as cc: + for task in self.filter_processes(): + cc.SwitchProcessContext(task) + + for vaddr in self.addresses: + self.render_address(renderer, vaddr) + + else: + # Use current process context. + for vaddr in self.addresses: + self.render_address(renderer, vaddr) + + def render_address(self, renderer, vaddr): + renderer.section(name="{0:#08x}".format(vaddr)) + self.address_space = self.session.GetParameter("default_address_space") renderer.format("Virtual {0:#08x} Page Directory 0x{1:08x}\n", - self.address, self.address_space.dtb) + vaddr, self.address_space.dtb) - for name, value, address in self.vtop(self.address, self.address_space): + for name, value, address in self.vtop(vaddr, self.address_space): if address: # Properly format physical addresses. renderer.format( @@ -281,16 +291,16 @@ def render(self, renderer): renderer.format("{0}\n", name) if name == "pde" and not value & 1: - self.render_pte(0, value, renderer) + self.render_pte(0, value, renderer, vaddr) break if name == "pte": - self.render_pte(address, value, renderer) + self.render_pte(address, value, renderer, vaddr) break # The below re-does all the analysis using the address space. It should # agree! - physical_address = self.address_space.vtop(self.address) + physical_address = self.address_space.vtop(vaddr) if physical_address is None: renderer.format("Physical Address Invalid\n") else: diff --git a/rekall/plugins/windows/pfn_test.py b/rekall/plugins/windows/pfn_test.py index e284b77e4..e875d3c04 100644 --- a/rekall/plugins/windows/pfn_test.py +++ b/rekall/plugins/windows/pfn_test.py @@ -25,6 +25,20 @@ from rekall import testlib +class TestVtoP(testlib.SimpleTestCase): + # Create a test case by running the vadmap plugin and selecting at least one + # virtual address from each type. + PARAMETERS = dict( + commandline="vtop --pid %(pid)s - %(vaddr)s", + vaddr="0x00010000 0x00036000" + ) + +class TestPTE(testlib.SimpleTestCase): + PARAMETERS = dict( + commandline="pte %(pte)s", + pte="0x3286b8" + ) + class TestPFN(testlib.RekallBaseUnitTestCase): """Test the pfn module.""" diff --git a/rekall/plugins/windows/ssdt.py b/rekall/plugins/windows/ssdt.py index 86f08b9f1..89918cd22 100644 --- a/rekall/plugins/windows/ssdt.py +++ b/rekall/plugins/windows/ssdt.py @@ -34,7 +34,7 @@ class WinSSDT(common.WindowsCommandPlugin): def _find_process_context(self, table_ptr, cc): for proc in self.session.plugins.pslist( - proc_regex="csrss").filter_processes(): + proc_regex="csrss").filter_processes(): cc.SwitchProcessContext(proc) table_ptr.obj_vm = self.session.GetParameter( @@ -77,17 +77,20 @@ def render(self, renderer): with cc: for i, descriptor in enumerate(ssdt.Descriptors): table_ptr = descriptor.KiServiceTable - if not table_ptr.is_valid(): + + # Sometimes the table is not mapped. In this case we need to + # find a process context which maps the win32k.sys driver. + if table_ptr[0] == 0: table_ptr = self._find_process_context(table_ptr, cc) - table = table_ptr.deref() - renderer.section("Table %s @ %#x" % (i, table_ptr.v())) + renderer.section( + "Table %s @ %#x" % (i, table_ptr[0].obj_offset)) renderer.table_header([("Entry", "entry", "[addr]"), ("Target", "target", "[addrpad]"), ("Symbol", "symbol", "")]) if self.profile.metadata("arch") == "AMD64": - self._render_x64_table(table, renderer) + self._render_x64_table(table_ptr, renderer) else: - self._render_x86_table(table, renderer) + self._render_x86_table(table_ptr, renderer) diff --git a/rekall/plugins/windows/vadinfo.py b/rekall/plugins/windows/vadinfo.py index 61f5de433..6f7dd6070 100644 --- a/rekall/plugins/windows/vadinfo.py +++ b/rekall/plugins/windows/vadinfo.py @@ -139,7 +139,7 @@ def write_vad_control(self, renderer, vad): renderer.format( "Control Flags: {0}\n", control_area.u.Flags) - file_object = control_area.FilePointer.dereference() + file_object = control_area.FilePointer if file_object and file_object != 0: renderer.format( "FileObject @{0:08x} FileBuffer @ {1:08x} , " @@ -151,7 +151,7 @@ def write_vad_ext(self, renderer, vad): if vad.obj_type != "_MMVAD_SHORT": renderer.format( "First prototype PTE: {0:08x} Last contiguous PTE: " - "{1:08x}\n", vad.FirstPrototypePte.obj_offset, + "{1:08x}\n", vad.FirstPrototypePte[0], vad.LastContiguousPte) renderer.format("Flags2: {0}\n", vad.u2.VadFlags2) diff --git a/rekall/plugins/windows/vadinfo_test.py b/rekall/plugins/windows/vadinfo_test.py index d4ef2c5a4..876c9e1f0 100644 --- a/rekall/plugins/windows/vadinfo_test.py +++ b/rekall/plugins/windows/vadinfo_test.py @@ -41,6 +41,14 @@ class TestVADWalk(testlib.SimpleTestCase): ) +class TestVADMap(testlib.SimpleTestCase): + """Test VADMap.""" + + PARAMETERS = dict( + commandline="vadmap --pid %(pid)s" + ) + + class TestVad(testlib.RekallBaseUnitTestCase): """Test the vad module.""" diff --git a/rekall/registry.py b/rekall/registry.py index 5e6ad2fd8..c1dc73d96 100644 --- a/rekall/registry.py +++ b/rekall/registry.py @@ -36,8 +36,6 @@ __author__ = "Michael Cohen " -import logging - class classproperty(property): """A property that can be called on classes.""" diff --git a/rekall/rekal.py b/rekall/rekal.py index 27719e5ce..03a71d2bd 100755 --- a/rekall/rekal.py +++ b/rekall/rekal.py @@ -61,8 +61,7 @@ def main(argv=None): try: # Run the plugin with plugin specific args. - user_session.RunPlugin(plugin_cls, **config.RemoveGlobalOptions( - vars(flags))) + user_session.RunPlugin(plugin_cls, **config.RemoveGlobalOptions(flags)) except Exception as e: if getattr(flags, "debug", None): pdb.post_mortem(sys.exc_info()[2]) diff --git a/rekall/scan.py b/rekall/scan.py index 7d6fdacd5..f4db9085c 100644 --- a/rekall/scan.py +++ b/rekall/scan.py @@ -107,7 +107,7 @@ def scan(self, offset=0, maxlen=None): Args: offset: The starting offset in our current address space to scan. - maxlen: The maximum length to scan. If no provided we just scan until + maxlen: The maximum length to scan. If not provided we just scan until there is no data. Yields: @@ -191,7 +191,7 @@ def scan(self, offset=0, maxlen=None): # Consume the next block in this range. buffer_as.assign_buffer( - overlap + self.address_space.base.read( + overlap + self.session.physical_address_space.read( phys_chunk_offset, chunk_size), base_offset=chunk_offset - len(overlap)) diff --git a/rekall/session.py b/rekall/session.py index 495fb5044..8bcd359d0 100644 --- a/rekall/session.py +++ b/rekall/session.py @@ -49,7 +49,7 @@ config.DeclareOption( - "--profile_path", default=[], action="append", + "--profile_path", default=[], type="ArrayStringParser", help="Path to search for profiles. This can take " "any form supported by the IO Manager (e.g. zip files, " "directories, URLs etc)") @@ -59,7 +59,7 @@ config.DeclareOption( "--buffer_size", default=20*1024*1024, - action=config.IntParser, + type="IntParser", help="The maximum size of buffers we are allowed to read. " "This is used to control Rekall memory usage.") @@ -68,7 +68,7 @@ help="If specified we write output to this file.") config.DeclareOption( - "--max_collector_cost", default=4, type=int, + "--max_collector_cost", default=4, type="IntParser", help="If specified, collectors with higher cost will not be used.") @@ -228,8 +228,9 @@ def _set_filename(self, filename, parameters): - Update the filename - Reload the profile and possibly autodetect it. """ - # This is used by the ipython prompt. - self.Set('base_filename', os.path.basename(filename)) + if filename: + # This is used by the ipython prompt. + self.Set('base_filename', os.path.basename(filename)) # Reset any caches. if self.session: diff --git a/rekall/testlib.py b/rekall/testlib.py index 6814e75f1..9761cecf4 100644 --- a/rekall/testlib.py +++ b/rekall/testlib.py @@ -173,6 +173,7 @@ def LaunchExecutable(self, config_options): return {} if baseline_commandline: + baseline_commandline = "- %s" % baseline_commandline for k, v in config_options.items(): # prepend all global options to the command line. if k.startswith("-"): @@ -379,6 +380,18 @@ def testCase(self): self.assertEqual(previous, current) +class SortedComparison(SimpleTestCase): + + __abstract = True + + def testCase(self): + previous = sorted(self.baseline['output']) + current = sorted(self.current['output']) + + # Compare the entire table + self.assertListEqual(previous, current) + + class DisabledTest(RekallBaseUnitTestCase): """Disable a test.""" disabled = True diff --git a/rekall/ui/json_renderer.py b/rekall/ui/json_renderer.py index 9d16a23ab..02fd85e29 100644 --- a/rekall/ui/json_renderer.py +++ b/rekall/ui/json_renderer.py @@ -235,7 +235,7 @@ def EncodeToJsonSafe(self, item, details=False, **options): # Store the mro of the item. state["mro"] = self.get_mro(item) - # Store an object ID for this item to ensure that the decoded can re-use + # Store an object ID for this item to ensure that the decoder can re-use # objects if possible. The ID is globally unique for this object and # does not change. try: @@ -270,7 +270,7 @@ def GetState(self, item, **_): name=unicode(item.obj_name), vm=item.obj_vm, profile=item.obj_profile - ) + ) class BaseAddressSpaceObjectRenderer(StateBasedObjectRenderer): diff --git a/rekall/ui/renderer.py b/rekall/ui/renderer.py index 0f0f44cd1..e010ac457 100644 --- a/rekall/ui/renderer.py +++ b/rekall/ui/renderer.py @@ -95,15 +95,15 @@ config.DeclareOption( - "-v", "--verbose", default=False, action="store_true", + "-v", "--verbose", default=False, type="Boolean", help="Set logging to debug level.", group="Output control") config.DeclareOption( - "-q", "--quiet", default=False, action="store_true", + "-q", "--quiet", default=False, type="Boolean", help="Turn off logging to stderr.", group="Output control") config.DeclareOption( - "--debug", default=False, action="store_true", + "--debug", default=False, type="Boolean", help="If set we break into the debugger on error conditions.") diff --git a/rekall/ui/text.py b/rekall/ui/text.py index ba14c422b..0176e8113 100644 --- a/rekall/ui/text.py +++ b/rekall/ui/text.py @@ -49,11 +49,11 @@ help="The pager to use when output is larger than a screen full.") config.DeclareOption( - "--paging_limit", default=None, group="Interface", type=int, + "--paging_limit", default=None, group="Interface", type="IntParser", help="The number of output lines before we invoke the pager.") config.DeclareOption( - "--nocolors", default=False, action="store_true", group="Interface", + "--nocolors", default=False, type="Boolean", group="Interface", help="If set suppress outputting colors.") @@ -431,8 +431,8 @@ def tparm(self, capabilities, *args): def Render(self, target, foreground=None, background=None): """Decorate the string with the ansii escapes for the color.""" if (not self.terminal_capable or - foreground not in self.COLOR_MAP or - foreground not in self.COLOR_MAP): + foreground not in self.COLOR_MAP or + foreground not in self.COLOR_MAP): return utils.SmartUnicode(target) escape_seq = "" @@ -589,6 +589,9 @@ def wrap(cls, value, width): result.lines.append( wrapped_line + " " * (width - len(wrapped_line))) + if not result.lines: + result.lines = [""] + result.Justify() return result @@ -781,7 +784,7 @@ def render_header(self): """Returns a Cell formed by joining all the column headers.""" # Get each column to write its own header and then we join them all up. return Cell.Join(*[c.render_header() for c in self.columns], - tablesep=self.options.get("tablesep", " ")) + tablesep=self.options.get("tablesep", " ")) def get_row(self, *row, **options): """Format the row into a single Cell spanning all output columns. @@ -795,7 +798,7 @@ def get_row(self, *row, **options): """ return Cell.Join( *[c.render_row(x, **options) for c, x in zip(self.columns, row)], - tablesep=self.options.get("tablesep", " ")) + tablesep=self.options.get("tablesep", " ")) def render_row(self, row=None, highlight=None, annotation=False, **options): """Write the row to the output.""" @@ -1001,11 +1004,14 @@ def open(self, directory=None, filename=None, mode="rb"): if filename is None and directory is None: raise IOError("Must provide a filename") + if directory is None: + directory, filename = os.path.split(filename) + filename = utils.SmartStr(filename) or "Unknown%s" % self._object_id # Filter the filename for illegal chars. filename = re.sub( - r"[^a-zA-Z0-9_.{}\[\]\- ]", + r"[^a-zA-Z0-9_.@{}\[\]\- ]", lambda x: "%" + x.group(0).encode("hex"), filename) diff --git a/setup.py b/setup.py index aadb837a4..d02d0e8d7 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ "pytz >= 2012", "ipython >= 2.0.0", "pycrypto >= 2.3.1", - "pyelftools >= 0.21", + "pyelftools >= 0.22", "distorm3 >= 0", "acora >= 1.8", "codegen >= 1.0", diff --git a/tools/testing/test_suite.py b/tools/testing/test_suite.py index 35a9015d4..bbf0f0ff2 100755 --- a/tools/testing/test_suite.py +++ b/tools/testing/test_suite.py @@ -383,6 +383,7 @@ def RunTests(self): if self.FLAGS.tests and plugin_cls.__name__ not in self.FLAGS.tests: continue + if plugin_cls.disabled: continue