Base for payloads with offsets and limit (refs xmendez#343)

framework/plugins/api.py

@@ -7,6 +7,8 @@
 import urlparse
 import urllib2
 import json
+import abc
+import itertools
 
 # Util methods when processing fuzz results 
 
@@ -28,37 +30,6 @@ def url_filename_ext(url):
     return ext
 
 # Util methods for accessing search results
-def search_bing(dork, key = None, raw = False):
-    if key is None:
-	key = Facade().sett.get('plugins', 'bing_apikey')
-
-    if not key:
-	raise FuzzException(FuzzException.FATAL, "An api Bing key is needed. Please chek wfuzz.ini.")
-
-    # some code taken from http://www.securitybydefault.com/2014/07/search2auditpy-deja-que-bing-haga-el.html?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+SecurityByDefault+%28Security+By+Default%29
-    user_agent = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; FDM; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 1.1.4322)'
-    creds = (':%s' % key).encode('base64')[:-1]
-    auth = 'Basic %s' % creds
-
-    # temporary solution, wf should have a process performing http requests. even plugins might need this.
-
-    try:
-	request = urllib2.Request('https://api.datamarket.azure.com/Data.ashx/Bing/Search/Composite?Sources=%27web%27&Query=%27'+dork+'%27&$format=json')
-	request.add_header('Authorization', auth)
-	request.add_header('User-Agent', user_agent)
-	requestor = urllib2.build_opener()
-	result = requestor.open(request)
-    except Exception, e:
-	raise FuzzException(FuzzException.FATAL, "Error when retrieving Bing API results: %s." % e.msg)
-
-    results = json.loads(result.read())
-    #test results = {u'd': {u'results': [{u'Web': [{u'Description': u'Diario de informaci\xf3n general de USA, noticias de \xfaltima hora de USA, el mundo, local, deportes, noticias curiosas y m\xe1s', u'Title': u'20minutos.com - El medio social - \xdaltima hora en USA y el ...', u'Url': u'http://www.20minutos.com/', u'__metadata': {u'type': u'WebResult', u'uri': u"https://api.datamarket.azure.com/Data.ashx/Bing/Search/ExpandableSearchResultSet(guid'b670a6b6-6ae7-4830-ad6f-83b525d6266d')/Web?$skip=0&$top=1"}, u'DisplayUrl': u'www.20minutos.com', u'ID': u'546995b5-587a-4618-984d-93bc5041e067'}, {u'Description': u'Informaci\xf3n, noticias y resultados de deportes: F\xfatbol, Baloncesto, NBA, Beisbol, F\xf3rmula 1, MotoGP, Tenis y m\xe1s en 20minutos.com', u'Title': u'Noticias deportivas - 20minutos.com', u'Url': u'http://www.20minutos.com/deportes/', u'__metadata': {u'type': u'WebResult', u'uri': u"https://api.datamarket.azure.com/Data.ashx/Bing/Search/ExpandableSearchResultSet(guid'b670a6b6-6ae7-4830-ad6f-83b525d6266d')/Web?$skip=1&$top=1"}, u'DisplayUrl': u'www.20minutos.com/deportes', u'ID': u'2ff2cd36-eece-4810-9b00-cba7d5ecfa47'}], u'VideoTotal': u'', u'RelatedSearch': [], u'Image': [], u'__metadata': {u'type': u'ExpandableSearchResult', u'uri': u"https://api.datamarket.azure.com/Data.ashx/Bing/Search/Composite?Sources='web'&Query='ip:193.148.34.26'&$skip=0&$top=1"}, u'ImageOffset': u'', u'AlterationOverrideQuery': u'', u'ImageTotal': u'', u'WebTotal': u'20', u'SpellingSuggestionsTotal': u'', u'WebOffset': u'0', u'Video': [], u'News': [], u'AlteredQuery': u'', u'SpellingSuggestions': [], u'VideoOffset': u'', u'NewsTotal': u'', u'ID': u'b670a6b6-6ae7-4830-ad6f-83b525d6266d', u'NewsOffset': u''}]}}
-
-    if raw:
-	return results
-    else:
-	return results['d']['results'][0]['Web']
-
 class BingIter:
     def __init__(self, dork, offset = 0, limit = 0, key = None):
 	if key is None:
@@ -226,3 +197,53 @@ def __init__(self):
 
     def blacklisted_extension(self, url):
 	return url_filename_ext(url) in self.black_list
+
+# Payloads specializations with common methods useful for their own type
+
+class OffsetPayload:
+    __metaclass__ = abc.ABCMeta 
+
+    def __init__(self, default_param, extra_params):
+	offset = 0
+	limit = 0
+	if extra_params:
+	    if extra_params.has_key("offset"):
+		offset = int(extra_params["offset"])
+
+	    if extra_params.has_key("limit"):
+		limit = int(extra_params["limit"])
+
+	is_sliced, self._iterator = self.my_slice_iter(default_param, offset, limit)
+	self._slice_it(is_sliced, offset, limit)
+
+	if self._count <= 0:
+	    raise FuzzException(FuzzException.FATAL, "Number of elements is negative.")
+
+    def _slice_it(self, is_sliced, offset, limit):
+	maxc = self.my_max_count()
+
+	if not is_sliced:
+	    if offset > maxc: offset = maxc
+	    if limit == 0: limit = maxc
+
+	    self._iterator = itertools.islice(self._iterator, offset, min(offset + limit, maxc))
+	    self._count = min(offset + limit, maxc) - offset
+	else:
+	    self._count = maxc
+
+    @abc.abstractmethod
+    def my_max_count(self):
+	return
+
+    @abc.abstractmethod
+    def my_slice_iter(self, param, offset, limit):
+	return
+
+    def next (self):
+	return self._iterator.next().strip()
+
+    def count(self):
+	return self._count
+
+    def __iter__ (self):
+	return self

plugins/payloads.py

@@ -7,33 +7,35 @@
 from externals.moduleman.plugin import moduleman_plugin
 from framework.core.myexception import FuzzException
 from framework.fuzzer.base import wfuzz_iterator
-from framework.plugins.api import search_bing
+from framework.plugins.api import BingIter
+from framework.plugins.api import OffsetPayload
 
 @wfuzz_iterator
 @moduleman_plugin("count", "next", "__iter__")
-class file:
+class file(OffsetPayload):
     name = "file"
     description = "Returns each word from a file."
     category = ["default"]
     priority = 99
 
-    def __init__(self, filename, extra):
-	try:
-	    self.f = open(filename,"r")
-	except IOError:
-	    raise FuzzException(FuzzException.FATAL, "Error opening file")
 
-	self.__count = len(self.f.readlines())
-	self.f.seek(0)
+    def __init__(self, default_param, extra):
+	OffsetPayload.__init__(self, default_param, extra)
 
-    def next (self):
-	return self.f.next().strip()
+    def my_max_count(self):
+	return self.__max
 
-    def count(self):
-	return self.__count
+    def my_slice_iter(self, default_param, offset, limit):
+	maxl = 0
 
-    def __iter__ (self):
-	return self
+	try:
+	    f = open(default_param, "r")
+	    self.__max = len(f.readlines())
+	    f.seek(0)
+	except IOError:
+	    raise FuzzException(FuzzException.FATAL, "Error opening file")
+
+	return False, f
 
 @wfuzz_iterator
 @moduleman_plugin("count", "next", "__iter__")
@@ -369,7 +371,7 @@ def xcombinations(self, items, n):
 
 @wfuzz_iterator
 @moduleman_plugin("count", "next", "__iter__")
-class bing:
+class bing(OffsetPayload):
     '''
     Some examples of bing hacking:
     - http://www.elladodelmal.com/2010/02/un-poco-de-bing-hacking-i-de-iii.html
@@ -378,53 +380,47 @@ class bing:
     description = "Returns URL results of a given bing API search (needs api key). ie, intitle:\"JBoss JMX Management Console\"-10"
     category = ["default"]
     priority = 99
+    def __init__(self, default_param, extra):
+	OffsetPayload.__init__(self, default_param, extra)
 
-    def __init__(self, dork, extra):   
-	self.l = search_bing(dork)
-	self.__count = len(self.l)
-	self.current = 0
-
-    def __iter__ (self):
-	return self
+    def my_slice_iter(self, default_param, offset, limit):
+	itera = BingIter(default_param, offset, limit)
+	self.__max = itera.max_count
 
-    def count(self):
-	return self.__count
+	return True, itera
 
-    def next (self):
-	if self.current >= self.__count:
-	    raise StopIteration
-	else:
-	    elem = self.l[self.current]['Url']
-	    self.current += 1
-	    return str(elem.strip())
+    def my_max_count(self):
+	return self.__max
 
 @wfuzz_iterator
 @moduleman_plugin("count", "next", "__iter__")
-class wfuzz:
+class wfuzz(OffsetPayload):
     name = "wfuzz"
     description = "Returns fuzz results' URL from a previous stored wfuzz session."
     category = ["default"]
     priority = 99
 
-    def __init__(self, path, extra):   
+    def __init__(self, default_param, extra):   
+	OffsetPayload.__init__(self, default_param, extra)
+	self.__max = 0
+
+    def my_max_count(self):
+	return self.__max
+
+    def my_slice_iter(self, default_param, offset, limit):
 	pkl_file = None
 	try:
-	    pkl_file = gzip.open(path, 'r+b')
+	    pkl_file = gzip.open(default_param, 'r+b')
 	    #pkl_file = open(path, 'r+b')
-	    self.fuzz_results = pickle.load(pkl_file)
+	    fuzz_results = pickle.load(pkl_file)
 	except Exception,e:
 	    raise FuzzException(FuzzException.FATAL, "Error opening wfuzz results file: %s" % str(e))
 	finally:
 	    if pkl_file: pkl_file.close()
 
-	self.__count = len(self.fuzz_results)
-	self.fuzz_results = iter(self.fuzz_results)
+	self.__max = len(fuzz_results)
 
-    def __iter__ (self):
-	return self
-
-    def count(self):
-	return self.__count
+	return False, fuzz_results
 
     def next (self):
-	return self.fuzz_results.next().url
+	return self._iterator.next().url