slice filter for payloads (fixes xmendez#343)

framework/core/options.py

@@ -29,6 +29,7 @@ def _defaults(self):
 		sl = [],
 		sh = [],
 		filterstr = "",
+		slicestr = "",
 		),
 	    payload_options = dict(
 		payloads = [],
@@ -150,6 +151,7 @@ class FuzzSession:
     def __init__(self):
 	self._values = {
 	    "filter_params": None,
+	    "slice_params": None,
 	    "printer_tool": Facade().sett.get('general', 'default_printer'),
 	    "rlevel": 0,
 	    "script_string": "",
@@ -175,7 +177,9 @@ def get(self, name):
     def from_options(options):
 	fuzz_options = FuzzSession()
 
+        # filter options
 	fuzz_options.set("filter_params", FuzzResFilter.from_options(options["filter_options"]))
+	fuzz_options.set("slice_params", FuzzResFilter(filter_string = options["filter_options"]['slicestr']))
 
 	# conn options
 	fuzz_options.set('proxy_list', options["conn_options"]["proxy_list"])

framework/fuzzer/Fuzzer.py

@@ -19,6 +19,7 @@
 from framework.plugins.jobs import ProcessorQ
 from framework.plugins.jobs import RoundRobin
 from framework.fuzzer.filter import FilterQ
+from framework.fuzzer.filter import SliceQ
 
 from externals.reqresp.exceptions import ReqRespException
 from externals.reqresp.cache import HttpCache
@@ -113,9 +114,10 @@ def __init__(self, options):
 
 	recursive = lplugins or options.get("rlevel") > 0
 	filtering = options.get('filter_params').is_active()
+	slicing = options.get('slice_params').is_active()
 
 	# Create queues (in reverse order)
-	# genReq ---> seed_queue -> http_queue -> [round_robin] -> [plugins_queue] * N -> process_queue -> [routing_queue] -> [filter_queue]---> results_queue
+	# genReq ---> seed_queue -> [slice_queue] -> http_queue -> [round_robin] -> [plugins_queue] * N -> process_queue -> [routing_queue] -> [filter_queue]---> results_queue
 	self.results_queue = MyPriorityQueue()
 	self.filter_queue = FilterQ(options.get("filter_params"), self.results_queue) if filtering else None
 	self.routing_queue = RoutingQ(None, self.filter_queue if filtering else self.results_queue) if recursive else None
@@ -129,7 +131,8 @@ def __init__(self, options):
 	    self.http_queue = DryRunQ(self.plugins_queue if lplugins else self.process_queue)
 	else:
 	    self.http_queue = HttpQueue(options, self.plugins_queue if lplugins else self.process_queue)
-	self.seed_queue = SeedQ(self.genReq, options.get("sleeper"), self.http_queue)
+	self.slice_queue = SliceQ(options.get("slice_params"), self.http_queue) if slicing else None
+	self.seed_queue = SeedQ(self.genReq, options.get("sleeper"), self.slice_queue if slicing else self.http_queue)
 
 	# recursion routes
 	if recursive:
@@ -152,7 +155,7 @@ def process(self):
 	self.results_queue.task_done()
 
 	if isinstance(item, FuzzResult):
-	    self.genReq.stats.processed += 1
+	    if item.is_processable: self.genReq.stats.processed += 1
 	    self.genReq.stats.pending_fuzz -= 1
 	    if not item.is_visible: self.genReq.stats.filtered += 1 
 	elif isinstance(item, FuzzException) and item.etype == FuzzException.SIG_ENDSEED:
@@ -169,7 +172,7 @@ def process(self):
     def next(self):
 	# ignore end seed marks
 	res = self.process()
-	while isinstance(res, FuzzException) and res.etype == FuzzException.SIG_ENDSEED:
+	while (isinstance(res, FuzzResult) and not res.is_processable) or (isinstance(res, FuzzException) and res.etype == FuzzException.SIG_ENDSEED):
 	    res = self.process()
 
 	# done! (None sent has gone through all queues).

framework/fuzzer/filter.py

@@ -14,7 +14,7 @@
 
 
 class FuzzResFilter:
-    def __init__(self, ffilter = None):
+    def __init__(self, ffilter = None, filter_string = None):
 	if PYPARSING:
 	    element = oneOf("c code l lines w words h chars i index")
 	    adv_element = oneOf("intext inurl site inheader filetype")
@@ -36,6 +36,9 @@ def __init__(self, ffilter = None):
 	    nestedformula.setParseAction(self.__compute_formula)
 	    self.finalformula.setParseAction(self.__myreduce)
 
+        if ffilter is not None and filter_string is not None:
+            raise FuzzException(FuzzException.FATAL, "A filter must be initilized with a filter string or an object, not both")
+
 	self.res = None
         if ffilter:
             self.hideparams = ffilter
@@ -51,6 +54,9 @@ def __init__(self, ffilter = None):
                 filter_string = ""
                 )
 
+        if filter_string:
+            self.hideparams['filter_string'] = filter_string
+
 	if "XXX" in self.hideparams['codes']:
 	    self.hideparams['codes'].append("0")
 
@@ -251,8 +257,6 @@ def __init__(self, ffilter, queue_out):
 	Thread.__init__(self)
 
 	self.setName('filter_thread')
-
-	self.queue_out = queue_out
 	self.ffilter = ffilter
 
     def get_name(self):
@@ -267,22 +271,20 @@ def process(self, prio, item):
 	item.is_visible = self.ffilter.is_visible(item)
 	self.send(item)
 
-if __name__ == "__main__":
-    tests = []
-    tests.append("(w=200 and w=200) or w=200")
-    tests.append("(w=400 and w=200) and (w=200 or w=200 or w=000)")
-    tests.append("(w=200 and l=7) and (h=23)")
-    tests.append("w=201")
-    tests.append("w=200")
-
-    class t:
-	code = 200
-	words = 200
-	lines = 7
-	chars = 23
-
-    res = t()
-
-    f = FilterQ()
-    for i in tests:
-	print "%s := %s" % (str(i), f.is_visible(res, i))
+class SliceQ(FuzzQueue):
+    def __init__(self, ffilter, queue_out):
+	FuzzQueue.__init__(self, queue_out)
+	Thread.__init__(self)
+
+	self.setName('slice_thread')
+	self.ffilter = ffilter
+
+    def get_name(self):
+	return 'slice_thread'
+
+    def _cleanup(self):
+	pass
+
+    def process(self, prio, item):
+	item.is_processable = self.ffilter.is_visible(item)
+	self.send(item)

framework/fuzzer/fuzzobjects.py

@@ -579,6 +579,7 @@ def __init__(self, history, exception = None):
         self.description = ""
         self.is_baseline = False
 	self.is_visible = True
+	self.is_processable = True
         self.rlevel = 1
         self.nres = 0 if self.is_baseline else FuzzResult.newid()
 
@@ -643,6 +644,7 @@ def from_soft_copy(self):
         fr.description = self.description
         fr.is_baseline = self.is_baseline
 	fr.is_visible = self.is_visible
+	fr.is_processable = self.is_processable
 	fr.type = self.type
         fr.rlevel = self.rlevel
 

framework/plugins/api/payloadtools.py

@@ -1,4 +1,3 @@
-from framework.fuzzer.filter import FuzzResFilter
 from framework.core.myexception import FuzzException
 
 import itertools
@@ -49,37 +48,6 @@ def next(self):
 
 	return str(attr)
 
-def filter_results(extra_params, itera):
-    ffilter = None
-
-    if extra_params and extra_params.has_key("filter"):
-	ffilter = FuzzResFilter()
-	ffilter.hideparams["filter_string"] = extra_params["filter"]
-
-	return itertools.ifilter(lambda x:ffilter.is_visible(x), itera)
-    else:
-	#raise FuzzException(FuzzException.FATAL, "Missing filter parameter in payload")
-	return itera
-
-def range_results(extra_params, itera):
-    offset = None
-    limit = None
-
-    try:
-	if extra_params:
-	    if extra_params.has_key("offset"):
-		offset = int(extra_params["offset"])
-
-	    if extra_params.has_key("limit"):
-		limit = int(extra_params["limit"])
-    except ValueError:
-	raise FuzzException(FuzzException.FATAL, "Invalid parameters, limit and offset should be numeric values.")
-
-    if offset is None and limit is None:
-	return itera
-    else:
-	if offset is not None and limit is not None: limit += offset
-	return itertools.islice(itera, offset, limit)
 
 class BingIter:
     def __init__(self, dork, offset = 0, limit = 0, key = None):

framework/ui/console/clparser.py

@@ -38,7 +38,7 @@ def show_plugins_help(self, registrant, cols=3, category="$all$"):
     def parse_cl(self):
 	# Usage and command line help
 	try:
-	    opts, args = getopt.getopt(self.argv[1:], "hAZIXvcb:e:R:d:z:r:f:t:w:V:H:m:o:s:p:w:",['zE=','oF=','recipe=', 'dump-recipe', 'req-delay=','conn-delay=','sc=','sh=','sl=','sw=','ss=','hc=','hh=','hl=','hw=','hs=','ntlm=','basic=','digest=','follow','script-help=','script=','script-args=','filter=','interact','help','version','dry-run'])
+	    opts, args = getopt.getopt(self.argv[1:], "hAZIXvcb:e:R:d:z:r:f:t:w:V:H:m:o:s:p:w:",['zE=','oF=','recipe=', 'dump-recipe', 'req-delay=','conn-delay=','sc=','sh=','sl=','sw=','ss=','hc=','hh=','hl=','hw=','hs=','ntlm=','basic=','digest=','follow','script-help=','script=','script-args=','slice=','filter=','interact','help','version','dry-run'])
 	    optsd = defaultdict(list)
 	    for i,j in opts:
 		optsd[i].append(j)
@@ -172,9 +172,15 @@ def _parse_filters(self, optsd, filter_params):
 	    sl = [],
 	    sh = [],
 	    filterstr = "",
+	    slicestr = "",
 	    ),
 	'''
 
+	if "--slice" in optsd:
+	    if not PYPARSING:
+		raise FuzzException(FuzzException.FATAL, "--filter switch needs pyparsing module.")
+	    filter_params['slicestr'] = optsd["--slice"][0]
+
 	if "--filter" in optsd:
 	    if not PYPARSING:
 		raise FuzzException(FuzzException.FATAL, "--filter switch needs pyparsing module.")

framework/ui/console/common.py

@@ -106,6 +106,8 @@
 \t			      A list of encoders can be used, ie. md5-sha1. Encoders can be chained, ie. md5@sha1.
 \t			      Encoders category can be used. ie. url
 \t--zE <params>		    : Extra arguments for a given payload (it must be preceded by -z).
+\t--slice <filter>	    : Filter payload\'s elements using the specified expression (Use BBB for taking values from baseline)
+\t			      It should be composed of: c,l,w,h,index,intext,inurl,site,inheader,filetype,ispath,hasquery;not,and,or;=,<,>,!=,<=,>=")
 \t-w wordlist		    : Specify a wordlist file (alias for -z file,wordlist).
 \t-V alltype		    : All parameters bruteforcing (allvars and allpost). No need for FUZZ keyword.
 \t-X			    : Payload within HTTP methods (ex: "FUZZ HTTP/1.0"). No need for FUZZ keyword.
@@ -119,7 +121,7 @@
 \t--sc/sl/sw/sh N[,N]+	    : Show responses with the specified code/lines/words/chars (Use BBB for taking values from baseline)
 \t--ss/hs regex		    : Show/Hide responses with the specified regex within the content
 \t--filter <filter>	    : Filter responses using the specified expression (Use BBB for taking values from baseline)
-\t			      It should be composed of: c,l,w,h,intext,inurl,site,inheader,filetype,ispath,hasquery;not,and,or;=,<,>,!=,<=,>=")
+\t			      It should be composed of: c,l,w,h,index,intext,inurl,site,inheader,filetype,ispath,hasquery;not,and,or;=,<,>,!=,<=,>=")
 \n%s
 ''' % (header_usage, examples_banner)
 

framework/utils/myqueue.py

@@ -4,6 +4,7 @@
 from Queue import PriorityQueue
 from threading import Thread
 from framework.core.myexception import FuzzException
+from framework.fuzzer.fuzzobjects import FuzzResult
 
 
 class MyPriorityQueue(PriorityQueue):
@@ -102,6 +103,10 @@ def run(self):
 		    self.send_first(item)
 		    self.task_done()
 		    continue
+                elif isinstance(item, FuzzResult) and not item.is_processable:
+		    self.send(item)
+		    self.task_done()
+		    continue
 
 		self.process(prio, item)
 		self.task_done()

plugins/payloads.py

@@ -7,7 +7,6 @@
 from framework.core.myexception import FuzzException
 from framework.fuzzer.base import wfuzz_iterator
 from framework.plugins.api.payloadtools import BingIter
-from framework.plugins.api.payloadtools import range_results, filter_results
 from framework.plugins.api.payloadtools import FuzzResPayload
 from framework.fuzzer.fuzzobjects import FuzzResult
 
@@ -18,32 +17,25 @@ class file:
     category = ["default"]
     priority = 99
 
-
-    def __init__(self, default_param, extra):
-	self.__max = -1
-	self.f = range_results(extra, self._my_gen(default_param))
-
-    def __iter__(self):
-	return self
-
-    def _my_gen(self, filename):
-	maxl = 0
-
+    def __init__(self, filename, extra):
 	try:
-	    f = open(filename, "r")
-	    self.__max = len(f.readlines())
-	    f.seek(0)
+	    self.f = open(filename,"r")
 	except IOError, e:
 	    raise FuzzException(FuzzException.FATAL, "Error opening file. %s" % str(e))
-
-	return f
 
-    def count(self):
-	return self.__max
+	self.__count = len(self.f.readlines())
+	self.f.seek(0)
 
-    def next(self):
+
+    def next (self):
 	return self.f.next().strip()
 
+    def count(self):
+	return self.__count
+
+    def __iter__(self):
+	return self
+
 
 @wfuzz_iterator
 class range:
@@ -411,7 +403,7 @@ class wfuzz(FuzzResPayload):
     def __init__(self, default_param, extra_params):
 	FuzzResPayload.__init__(self, default_param, extra_params)
 	self.__max = -1
-	self._it = range_results(extra_params, filter_results(extra_params, self._gen_wfuzz(default_param)))
+	self._it = self._gen_wfuzz(default_param)
 
     def __iter__(self, default_param, extra):   
 	return self