From b1bad0066347d99cbd09161efeb3970e4fbf23dd Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 15 Oct 2019 22:07:54 +0200 Subject: [PATCH 01/11] test to check cache with JSON HTTP request --- tests/test_reqresp.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_reqresp.py b/tests/test_reqresp.py index 7797cc2c..4999d0c0 100644 --- a/tests/test_reqresp.py +++ b/tests/test_reqresp.py @@ -237,6 +237,21 @@ def test_cache_key(self): fr.params.post = '' self.assertEqual(fr.to_cache_key(), 'http://www.wfuzz.org/-p') + def test_cache_key_json_header_before(self): + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/" + fr.params.post = '1' + fr.headers.request = {'Content-Type': 'application/json'} + + self.assertEqual(fr.to_cache_key(), 'http://www.wfuzz.org/-p1') + + def test_cache_key_json_header_after(self): + fr = FuzzRequest() + fr.headers.request = {'Content-Type': 'application/json'} + fr.url = "http://www.wfuzz.org/" + fr.params.post = '1' + + self.assertEqual(fr.to_cache_key(), 'http://www.wfuzz.org/-p1') if __name__ == '__main__': unittest.main() From 99ad783eddca960111c65e1bb0f78b55796b2f98 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 15 Oct 2019 22:47:53 +0200 Subject: [PATCH 02/11] add more get vars tests --- tests/test_reqresp.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_reqresp.py b/tests/test_reqresp.py index 4999d0c0..2c92ffbb 100644 --- a/tests/test_reqresp.py +++ b/tests/test_reqresp.py @@ -253,5 +253,25 @@ def test_cache_key_json_header_after(self): self.assertEqual(fr.to_cache_key(), 'http://www.wfuzz.org/-p1') + def test_cache_key_get_var(self): + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/?a&b=1" + + self.assertEqual(fr.to_cache_key(), 'http://www.wfuzz.org/-ga-gb') + + def test_get_vars(self): + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/?a&b=1" + self.assertEqual(fr.params.get, {'a': None, 'b': '1'}) + + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/?" + self.assertEqual(fr.params.get, {}) + + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/" + self.assertEqual(fr.params.get, {}) + + if __name__ == '__main__': unittest.main() From c33174cfa87221fdab162d9d889d38e63cfc5e9d Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 17:58:11 +0200 Subject: [PATCH 03/11] use raw post for handling post data as much as possible --- src/wfuzz/externals/reqresp/Request.py | 49 ++++++++-------- src/wfuzz/externals/reqresp/Variables.py | 10 ++-- src/wfuzz/fuzzobjects.py | 20 ++++--- tests/test_reqresp.py | 72 ++++++++++++++++++------ 4 files changed, 97 insertions(+), 54 deletions(-) diff --git a/src/wfuzz/externals/reqresp/Request.py b/src/wfuzz/externals/reqresp/Request.py index 90870662..9a7f9ea8 100644 --- a/src/wfuzz/externals/reqresp/Request.py +++ b/src/wfuzz/externals/reqresp/Request.py @@ -50,7 +50,7 @@ def __init__(self): self.multiPOSThead = {} self.__variablesGET = VariablesSet() - self.__variablesPOST = VariablesSet() + self._variablesPOST = VariablesSet() self._non_parsed_post = None # diccionario, por ejemplo headers["Cookie"] @@ -89,7 +89,7 @@ def __init__(self): @property def method(self): if self._method is None: - return "POST" if (self.getPOSTVars() or self._non_parsed_post is not None) else "GET" + return "POST" if self._non_parsed_post is not None else "GET" return self._method @@ -148,17 +148,14 @@ def __getattr__(self, name): elif name == "path": return self.__path elif name == "postdata": - if self._non_parsed_post is not None: - return self._non_parsed_post - if self.ContentType == "application/x-www-form-urlencoded": - return self.__variablesPOST.urlEncoded() + return self._variablesPOST.urlEncoded() elif self.ContentType == "multipart/form-data": - return self.__variablesPOST.multipartEncoded() + return self._variablesPOST.multipartEncoded() elif self.ContentType == 'application/json': - return self.__variablesPOST.json_encoded() + return self._variablesPOST.json_encoded() else: - return self.__variablesPOST.urlEncoded() + return self._variablesPOST.urlEncoded() else: raise AttributeError @@ -210,10 +207,10 @@ def existsGETVar(self, key): return self.__variablesGET.existsVar(key) def existPOSTVar(self, key): - return self.__variablesPOST.existsVar(key) + return self._variablesPOST.existsVar(key) def setVariablePOST(self, key, value): - v = self.__variablesPOST.getVariable(key) + v = self._variablesPOST.getVariable(key) v.update(value) # self._headers["Content-Length"] = str(len(self.postdata)) @@ -225,21 +222,25 @@ def getGETVars(self): return self.__variablesGET.variables def getPOSTVars(self): - return self.__variablesPOST.variables + return self._variablesPOST.variables def setPostData(self, pd, boundary=None): + self._non_parsed_post = pd + self._variablesPOST = VariablesSet() + try: - self.__variablesPOST = VariablesSet() - if self.ContentType == "application/x-www-form-urlencoded": - self.__variablesPOST.parseUrlEncoded(pd) - elif self.ContentType == "multipart/form-data": - self.__variablesPOST.parseMultipart(pd, boundary) + if self.ContentType == "multipart/form-data": + self._variablesPOST.parseMultipart(pd, boundary) elif self.ContentType == 'application/json': - self.__variablesPOST.parse_json_encoded(pd) + self._variablesPOST.parse_json_encoded(pd) else: - self.__variablesPOST.parseUrlEncoded(pd) + self._variablesPOST.parseUrlEncoded(pd) except Exception: - self._non_parsed_post = pd + try: + self._variablesPOST.parseUrlEncoded(pd) + except Exception: + print("Warning: POST parameters not parsed") + pass ############################################################################ @@ -345,8 +346,8 @@ def to_pycurl_object(c, req): else: c.setopt(pycurl.CUSTOMREQUEST, req.method) - if req.getPOSTVars() or req._non_parsed_post is not None: - c.setopt(pycurl.POSTFIELDS, python2_3_convert_to_unicode(req.postdata)) + if req._non_parsed_post is not None: + c.setopt(pycurl.POSTFIELDS, python2_3_convert_to_unicode(req._non_parsed_post)) c.setopt(pycurl.FOLLOWLOCATION, 1 if req.followLocation else 0) @@ -393,7 +394,7 @@ def perform(self): # ######## ESTE conjunto de funciones no es necesario para el uso habitual de la clase def getAll(self): - pd = self.postdata + pd = self._non_parsed_post if self._non_parsed_post else '' string = str(self.method) + " " + str(self.pathWithVariables) + " " + str(self.protocol) + "\n" for i, j in self._headers.items(): string += i + ": " + j + "\n" @@ -421,7 +422,7 @@ def parseRequest(self, rawRequest, prot="http"): tp = TextParser() tp.setSource("string", rawRequest) - self.__variablesPOST = VariablesSet() + self._variablesPOST = VariablesSet() self._headers = {} # diccionario, por ejemplo headers["Cookie"] tp.readLine() diff --git a/src/wfuzz/externals/reqresp/Variables.py b/src/wfuzz/externals/reqresp/Variables.py index 1416f95f..3ca34cf8 100644 --- a/src/wfuzz/externals/reqresp/Variables.py +++ b/src/wfuzz/externals/reqresp/Variables.py @@ -83,11 +83,11 @@ def parseUrlEncoded(self, cad): for i in cad.split("&"): if i: - list = i.split("=", 1) - if len(list) == 1: - dicc.append(Variable(list[0], None)) - elif len(list) == 2: - dicc.append(Variable(list[0], list[1])) + var_list = i.split("=", 1) + if len(var_list) == 1: + dicc.append(Variable(var_list[0], None)) + elif len(var_list) == 2: + dicc.append(Variable(var_list[0], var_list[1])) self.variables = dicc diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 0ecfe7d9..caaf30b7 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -18,7 +18,7 @@ from .filter import FuzzResFilter from .externals.reqresp import Request, Response -from .exception import FuzzExceptBadAPI, FuzzExceptBadOptions, FuzzExceptInternalError +from .exception import FuzzExceptBadAPI, FuzzExceptBadOptions, FuzzExceptInternalError, FuzzException from .facade import Facade, ERROR_CODE from .mixins import FuzzRequestUrlMixing, FuzzRequestSoupMixing @@ -114,19 +114,23 @@ def get(self, values): @property def post(self): - if self._req._non_parsed_post is None: - return params.param([(x.name, x.value) for x in self._req.getPOSTVars()]) - else: - return self._req.postdata + return params.param([(x.name, x.value) for x in self._req.getPOSTVars()]) @post.setter def post(self, pp): if isinstance(pp, dict): for key, value in pp.items(): self._req.setVariablePOST(key, str(value) if value is not None else value) + + self._req._non_parsed_post = self._req._variablesPOST.urlEncoded() + elif isinstance(pp, str): self._req.setPostData(pp) + @property + def raw_post(self): + return self._req._non_parsed_post + @property def all(self): return params.param(self.get + self.post) @@ -330,8 +334,8 @@ def update_from_raw_http(self, raw, scheme, raw_response=None, raw_content=None) self._request.parseRequest(raw, scheme) # Parse request sets postdata = '' when there's POST request without data - if self.method == "POST" and not self.params.post: - self.params.post = {'': None} + if self.method == "POST" and self.params.raw_post is None: + self.params.post = '' if raw_response: rp = Response() @@ -394,7 +398,7 @@ def from_copy(self): newreq.wf_ip = self.wf_ip newreq.headers.request = self.headers.request - newreq.params.post = self.params.post + newreq.params.post = self.params.raw_post newreq.follow = self.follow newreq.auth = self.auth diff --git a/tests/test_reqresp.py b/tests/test_reqresp.py index 2c92ffbb..2624ac9d 100644 --- a/tests/test_reqresp.py +++ b/tests/test_reqresp.py @@ -118,42 +118,56 @@ def test_seturl(self): fr.url = "www.wfuzz.org" self.assertEqual(fr.host, "www.wfuzz.org") - def test_setpostdata(self): + def test_empy_post(self): fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" - fr.params.post = 'a=1' + fr.params.post = '' self.assertEqual(fr.method, "POST") - self.assertEqual(fr.params.post, {'a': '1'}) + self.assertEqual(fr.params.post, {'': None}) + self.assertEqual(fr.params.raw_post, '') fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" - fr.params.post = '1' + fr.params.post = {} self.assertEqual(fr.method, "POST") - self.assertEqual(fr.params.post, {'1': None}) + self.assertEqual(fr.params.post, {}) + self.assertEqual(fr.params.raw_post, '') fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" - fr.params.post = '' + fr.params.post = None + self.assertEqual(fr.method, "GET") + self.assertEqual(fr.params.post, {}) + self.assertEqual(fr.params.raw_post, None) + + def test_setpostdata(self): + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/" + fr.params.post = 'a=1' self.assertEqual(fr.method, "POST") - self.assertEqual(fr.params.post, {'': None}) + self.assertEqual(fr.params.raw_post, 'a=1') + self.assertEqual(fr.params.post, {'a': '1'}) fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" - fr.params.post = {} - self.assertEqual(fr.method, "GET") - self.assertEqual(fr.params.post, {}) + fr.params.post = '1' + self.assertEqual(fr.method, "POST") + self.assertEqual(fr.params.post, {'1': None}) + self.assertEqual(fr.params.raw_post, '1') fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" fr.params.post = {'a': 1} self.assertEqual(fr.method, "POST") self.assertEqual(fr.params.post, {'a': '1'}) + self.assertEqual(fr.params.raw_post, 'a=1') fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" fr.params.post = {'a': '1'} self.assertEqual(fr.method, "POST") self.assertEqual(fr.params.post, {'a': '1'}) + self.assertEqual(fr.params.raw_post, 'a=1') fr = FuzzRequest() fr.url = "http://www.wfuzz.org/" @@ -161,13 +175,6 @@ def test_setpostdata(self): self.assertEqual(fr.method, "POST") self.assertEqual(fr.params.post, {"{'a': '1'}": None}) - fr = FuzzRequest() - fr.url = "http://www.wfuzz.org/" - fr.params.post = '1' - fr.headers.request = {'Content-Type': 'application/json'} - self.assertEqual(fr.method, "POST") - self.assertEqual(fr.params.post, {'1': None}) - def test_setgetdata(self): fr = FuzzRequest() @@ -272,6 +279,37 @@ def test_get_vars(self): fr.url = "http://www.wfuzz.org/" self.assertEqual(fr.params.get, {}) + def test_setpostdata_with_json(self): + fr = FuzzRequest() + fr.headers.request = {'Content-Type': 'application/json'} + fr.url = "http://www.wfuzz.org/" + fr.params.post = '{"string": "Foo bar","boolean": false}' + self.assertEqual(fr.params.post, {'string': 'Foo bar', 'boolean': False}) + + fr = FuzzRequest() + fr.headers.request = {'Content-Type': 'application/json'} + fr.url = "http://www.wfuzz.org/" + fr.params.post = '{"array": [1,2]}' + self.assertEqual(fr.params.post, {'array': [1, 2]}) + + def test_post_bad_json(self): + fr = FuzzRequest() + fr.headers.request = {'Content-Type': 'application/json'} + fr.url = "http://www.wfuzz.org/" + fr.params.post = '1' + + self.assertEqual(fr.method, "POST") + self.assertEqual(fr.params.post, {'1': None}) + self.assertEqual(fr.params.raw_post, '1') + + fr = FuzzRequest() + fr.url = "http://www.wfuzz.org/" + fr.headers.request = {'Content-Type': 'application/json'} + fr.params.post = 'a=1' + self.assertEqual(fr.method, "POST") + self.assertEqual(fr.params.raw_post, "a=1") + self.assertEqual(fr.params.post, {'a': '1'}) + if __name__ == '__main__': unittest.main() From 2bd7351da810f08dba7750b68475850e4323d08e Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 18:24:09 +0200 Subject: [PATCH 04/11] add python 3.7 to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ac295fec..e3b71733 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" before_install: - docker-compose -f tests/server_dir/docker-compose.yml up -d install: From 0b51cf2084bc8adafe1fa077797c6821d426279d Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 18:28:51 +0200 Subject: [PATCH 05/11] add libcurl to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e3b71733..80affbde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "3.6" - "3.7" before_install: + - apt install -y libcurl4-openssl-dev - docker-compose -f tests/server_dir/docker-compose.yml up -d install: - pip install coverage From 61363091ff32f43dcd573bd8f05bc4246d60ecfe Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 18:37:14 +0200 Subject: [PATCH 06/11] use apt addon --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 80affbde..f45803ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ python: - "3.6" - "3.7" before_install: - - apt install -y libcurl4-openssl-dev - docker-compose -f tests/server_dir/docker-compose.yml up -d install: - pip install coverage @@ -34,3 +33,7 @@ deploy: - /^v.*$/ tags: true python: 3.6 +addons: + apt: + packages: + - libcurl4-openssl-dev From 7b371211afec99f5ce078b23e15e1e5a6fdbfe3e Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 19:39:23 +0200 Subject: [PATCH 07/11] add python 3.7 to tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4089d33d..44ca51a6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = begin,docker,py27,py36,end +envlist = begin,docker,py27,py36,py37,end [testenv] commands = From 6222bb191aa9976ee410da651178b8d838b098ff Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 19:39:38 +0200 Subject: [PATCH 08/11] support generator in py37 --- src/wfuzz/plugins/payloads/wfuzzp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/payloads/wfuzzp.py b/src/wfuzz/plugins/payloads/wfuzzp.py index 21a1e6e5..c1c51a5d 100644 --- a/src/wfuzz/plugins/payloads/wfuzzp.py +++ b/src/wfuzz/plugins/payloads/wfuzzp.py @@ -60,4 +60,4 @@ def _gen_wfuzz(self, output_fn): except IOError as e: raise FuzzExceptBadFile("Error opening wfuzz payload file. %s" % str(e)) except EOFError: - raise StopIteration + return From 82b94ca87fe63393a5c2732db2ddf7bae28a1bc9 Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 20:03:15 +0200 Subject: [PATCH 09/11] bump version --- src/wfuzz/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/__init__.py b/src/wfuzz/__init__.py index 79ff961c..b0e2fb87 100644 --- a/src/wfuzz/__init__.py +++ b/src/wfuzz/__init__.py @@ -1,5 +1,5 @@ __title__ = 'wfuzz' -__version__ = "2.4" +__version__ = "2.4.1" __build__ = 0x023000 __author__ = 'Xavier Mendez' __license__ = 'GPL 2.0' From 844af33aaf1aa1169bddddb8e51c83aacfc49492 Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 20:11:33 +0200 Subject: [PATCH 10/11] update docs --- docs/user/advanced.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 56829100..9db3a4dd 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -508,7 +508,8 @@ headers.request.<> Specified HTTP request given header headers.response.<> Specified HTTP response given header params.all All HTTP request GET and POST parameters params.get All HTTP request GET parameters -params.post All HTTP request POST parameters +params.post HTTP request POST parameters in returned as a dictionary +params.raw_post HTTP request POST parameters payload params.get.<> Spcified HTTP request GET parameter params.post.<> Spcified HTTP request POST parameter pstrip Returns a signature of the HTTP request using the parameter's names without values (useful for unique operations) @@ -516,7 +517,7 @@ is_path Returns true when the HTTP request path refers to a reqtime Returns the total time that HTTP request took to be retrieved ============================ ============================================= -It is worth noting that Wfuzz will try to parse the POST parameters according to the specified content type header. Currently, application/x-www-form-urlencoded, multipart/form-dat and application/json are supported. +It is worth noting that Wfuzz will try to parse the POST parameters according to the specified content type header. Currently, application/x-www-form-urlencoded, multipart/form-dat and application/json are supported. This is prone to error depending on the data format, raw_post will not try to do any processing. FuzzRequest URL field is broken in smaller (read only) parts using the urlparse Python's module in the urlp attribute. From 53a1610fb861bdb30ff4e8497fa19417e1424caf Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 20 Oct 2019 20:41:35 +0200 Subject: [PATCH 11/11] nested json test --- tests/test_acceptance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index 086f2528..9d3e03ef 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -49,6 +49,8 @@ ("test_novalue_post_fuzz", "-z list --zD a -u {}/anything -d FUZZ".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --filter r.params.post.a:=1 --field r.params.post.a", ["1"], None), ("test_json_post_fuzz2", "-z list --zD anything -u {}/FUZZ -d {{\"a\":\"2\"}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.a", ["2"], None), ("test_json_post_fuzz3", "-z list --zD anything -u {}/FUZZ -d {{\"a\":\"2\"}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --filter r.params.post.a:=1 --field r.params.post.a", ["1"], None), + ("test_json_nested", "-z list --zD anything -u {}/FUZZ -d {{\"test\":\"me\",\"another\":1,\"nested\":{{\"this\":2}}}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.nested.this", [2], None), + ("test_json_nested2", "-z list --zD anything -u {}/FUZZ -d {{\"test\":\"me\",\"another\":1,\"nested\":{{\"this\":2}}}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.another", [1], None), # field fuzz values ("test_desc_fuzz", "-z range,1-1 {}/FUZZ".format(HTTPBIN_URL), "-z wfuzzp,$$PREVFILE$$ FUZZ", ["http://localhost:9000/1"], None), @@ -443,7 +445,7 @@ def duplicate_tests(test_list, group, test_gen_fun): def create_savedsession_tests(test_list, test_gen_fun): """ - generates wfuzz tests that run 2 times with recipe input, expecting same results. + generates wfuzz tests that run 2 times with a saved session, expecting same results. """ for test_name, prev_cli, next_cli, expected_res, exception_str in test_list: