From 18a608c0ff5ce61aa8ddf90776bc01b2310a2acd Mon Sep 17 00:00:00 2001 From: laramies Date: Thu, 2 Apr 2020 22:36:13 +0200 Subject: [PATCH 01/89] Update common.txt Added jira and confluence --- wordlist/general/common.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wordlist/general/common.txt b/wordlist/general/common.txt index ba228a38..9644af96 100644 --- a/wordlist/general/common.txt +++ b/wordlist/general/common.txt @@ -190,6 +190,7 @@ composer compressed comunicator con +confluence config configs configuration @@ -451,6 +452,7 @@ jdbc job join jrun +jira js jsp jsps From fc2c5fd7f75999d4887df4f284a82b3500f1e987 Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 23 Sep 2020 09:42:15 +0200 Subject: [PATCH 02/89] show diff when prev and verbose --- src/wfuzz/ui/console/mvc.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index a0fcb805..6c77f2c1 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -1,6 +1,7 @@ import sys from collections import defaultdict import threading +import difflib try: from itertools import zip_longest @@ -164,7 +165,7 @@ def __init__(self, session_options): self.term = Term() self.printed_lines = 0 - def _print_verbose(self, res, print_nres=True): + def _print_verbose(self, res, print_nres=True, extra_description=None): txt_colour = ( Term.noColour if not res.is_baseline or not self.colour else Term.fgCyan ) @@ -181,6 +182,10 @@ def _print_verbose(self, res, print_nres=True): if "Server" in res.history.headers.response: server = res.history.headers.response["Server"] + description = res.description + if extra_description: + description = "{} | diff:\n{}".format(res.description, extra_description) + rows = [ ("%09d:" % res.nres if print_nres else " |_", txt_colour), ("%.3fs" % res.timer, txt_colour), @@ -193,7 +198,7 @@ def _print_verbose(self, res, print_nres=True): ("%d Ch" % res.chars, txt_colour), (server, txt_colour), (location, txt_colour), - ('"%s"' % res.description, txt_colour), + ('"%s"' % description, txt_colour), ] self.term.set_colour(txt_colour) @@ -306,7 +311,15 @@ def result(self, res): ): prev_res = res.payload_man.get_payload_content(1) if self.verbose: - self._print_verbose(prev_res, print_nres=False) + delta = difflib.unified_diff( + prev_res.history.raw_content.splitlines(True), + res.history.raw_content.splitlines(True), + fromfile="prev", + tofile="current", + ) + self._print_verbose( + prev_res, print_nres=False, extra_description="".join(delta) + ) else: self._print(prev_res, print_nres=False) From abf79645a207bd523cf6d723c8a61d928b3374fc Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 24 Sep 2020 21:22:00 +0200 Subject: [PATCH 03/89] add prev,AA option to wfpayload --- src/wfuzz/ui/console/common.py | 2 ++ src/wfuzz/wfuzz.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/ui/console/common.py b/src/wfuzz/ui/console/common.py index cd609268..ac167cea 100644 --- a/src/wfuzz/ui/console/common.py +++ b/src/wfuzz/ui/console/common.py @@ -178,9 +178,11 @@ \t-v : Verbose information. \t-f filename,printer : Store results in the output file using the specified printer (raw printer if omitted). \t-o printer : Show results using the specified printer. +\t--prev : Print the previous HTTP requests (only when using payloads generating fuzzresults) \t--efield : Show the specified language expression together with the current payload. Repeat option for various fields. \t--field : Do not show the payload but only the specified language expression. Repeat option for various fields. \t +\t-A, --AA : Alias for --script=default,verbose -v -c \t--script= : Equivalent to --script=default \t--script= : Runs script's scan. is a comma separated list of plugin-files or plugin-categories \t--script-help= : Show help about scripts. diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index 41ccb552..1421fbb2 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -71,7 +71,7 @@ def usage(): from .api import fuzz try: - short_opts = "hvce:z:f:w:o:" + short_opts = "hvce:z:f:w:o:A" long_opts = [ "efield=", "ee=", @@ -100,6 +100,8 @@ def usage(): "script-help=", "script=", "script-args=", + "prev", + "AA", ] session_options = CLParser( sys.argv, From 58afda932ba8c09f8b1dfb3aa8ac9d1a6378b0a9 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 28 Sep 2020 21:07:14 +0200 Subject: [PATCH 04/89] fix payman when using fuzzres seed --- src/wfuzz/fuzzqueues.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index f77abc2c..939bab1c 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -7,7 +7,8 @@ from .factories.fuzzresfactory import resfactory from .factories.plugin_factory import plugin_factory -from .fuzzobjects import FuzzType, FuzzItem +from .factories.payman import payman_factory +from .fuzzobjects import FuzzType, FuzzItem, FuzzWord, FuzzWordType from .myqueues import FuzzQueue from .exception import ( FuzzExceptInternalError, @@ -17,7 +18,6 @@ ) from .myqueues import FuzzRRQueue from .facade import Facade -from .fuzzobjects import FuzzWordType from .ui.console.mvc import View @@ -413,7 +413,9 @@ def process(self, item): if item.payload_man.get_payload_type(1) == FuzzWordType.FUZZRES: item = item.payload_man.get_payload_content(1) item.update_from_options(self.options) - + item.payload_man = payman_factory.create( + "empty_payloadman", FuzzWord(item.url, FuzzWordType.WORD) + ) self.send(item) From d8b4752579441fa87f4593887841cd02fe7918bc Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 28 Sep 2020 21:07:21 +0200 Subject: [PATCH 05/89] links plugin to parse links on header --- src/wfuzz/plugins/scripts/links.py | 48 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 9c148d40..887e5d03 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -40,8 +40,13 @@ def __init__(self): ] self.regex = [] - for i in regex: - self.regex.append(re.compile(i, re.MULTILINE | re.DOTALL)) + for regex_str in regex: + self.regex.append(re.compile(regex_str, re.MULTILINE | re.DOTALL)) + + self.regex_header = [ + ('Link', re.compile(r'<(.*)>;')), + ('Location', re.compile(r'(.*)')), + ] self.add_path = self.kbase["links.add_path"] @@ -52,29 +57,36 @@ def __init__(self): ) def validate(self, fuzzresult): - return fuzzresult.code in [200] + self.list_links = set() + return fuzzresult.code in [200, 301, 302, 303, 307, 308] def process(self, fuzzresult): - list_links = set() # O # ParseResult(scheme='', netloc='', path='www.owasp.org/index.php/OWASP_EU_Summit_2008', params='', query='', fragment='') + for header, regex in self.regex_header: + if header in fuzzresult.history.headers.response: + for link_url in regex.findall(fuzzresult.history.headers.response[header]): + if link_url: + self.process_link(fuzzresult, link_url) + for regex in self.regex: for link_url in regex.findall(fuzzresult.history.content): - if not link_url: - continue - - parsed_link = parse_url(link_url) - - if ( - not parsed_link.scheme - or parsed_link.scheme == "http" - or parsed_link.scheme == "https" - ) and self.from_domain(fuzzresult, parsed_link): - cache_key = parsed_link.cache_key(self.base_fuzz_res.history.urlp) - if cache_key not in list_links: - list_links.add(cache_key) - self.enqueue_link(fuzzresult, link_url, parsed_link) + if link_url: + self.process_link(fuzzresult, link_url) + + def process_link(self, fuzzresult, link_url): + parsed_link = parse_url(link_url) + + if ( + not parsed_link.scheme + or parsed_link.scheme == "http" + or parsed_link.scheme == "https" + ) and self.from_domain(fuzzresult, parsed_link): + cache_key = parsed_link.cache_key(self.base_fuzz_res.history.urlp) + if cache_key not in self.list_links: + self.list_links.add(cache_key) + self.enqueue_link(fuzzresult, link_url, parsed_link) def enqueue_link(self, fuzzresult, link_url, parsed_link): # dir path From 3d8f3230aca14ab47452ab90679bba33c484db5a Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 28 Sep 2020 23:00:05 +0200 Subject: [PATCH 06/89] fix links test --- src/wfuzz/plugins/scripts/links.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 887e5d03..2568c1bf 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -31,7 +31,7 @@ def __init__(self): BasePlugin.__init__(self) regex = [ - r'href="((?!mailto:|tel:|#|javascript:).*?)"', + r'ref="((?!mailto:|tel:|#|javascript:).*?)"', r'src="((?!javascript:).*?)"', r'action="((?!javascript:).*?)"', # http://en.wikipedia.org/wiki/Meta_refresh @@ -55,6 +55,7 @@ def __init__(self): self.domain_regex = re.compile( self.kbase["links.regex"][0], re.MULTILINE | re.DOTALL ) + self.list_links = set() def validate(self, fuzzresult): self.list_links = set() From 275ff656d98afda154b4a510d5f9612455c0d8ad Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 28 Sep 2020 23:40:07 +0200 Subject: [PATCH 07/89] add word boundary to links re --- src/wfuzz/plugins/scripts/links.py | 14 ++++++++------ tests/plugins/test_links.py | 5 +++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 2568c1bf..e645410d 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -31,9 +31,9 @@ def __init__(self): BasePlugin.__init__(self) regex = [ - r'ref="((?!mailto:|tel:|#|javascript:).*?)"', - r'src="((?!javascript:).*?)"', - r'action="((?!javascript:).*?)"', + r'\b(?:(?', r'getJSON\("(.*?)"', @@ -44,8 +44,8 @@ def __init__(self): self.regex.append(re.compile(regex_str, re.MULTILINE | re.DOTALL)) self.regex_header = [ - ('Link', re.compile(r'<(.*)>;')), - ('Location', re.compile(r'(.*)')), + ("Link", re.compile(r"<(.*)>;")), + ("Location", re.compile(r"(.*)")), ] self.add_path = self.kbase["links.add_path"] @@ -67,7 +67,9 @@ def process(self, fuzzresult): for header, regex in self.regex_header: if header in fuzzresult.history.headers.response: - for link_url in regex.findall(fuzzresult.history.headers.response[header]): + for link_url in regex.findall( + fuzzresult.history.headers.response[header] + ): if link_url: self.process_link(fuzzresult, link_url) diff --git a/tests/plugins/test_links.py b/tests/plugins/test_links.py index a9408d08..553fbfe8 100644 --- a/tests/plugins/test_links.py +++ b/tests/plugins/test_links.py @@ -7,6 +7,11 @@ @pytest.mark.parametrize( "example_full_fuzzres_content, expected_links", [ + (b'\n', [],), + ( + b'\n', + ["http://www.wfuzz.org/1.json", "http://www.wfuzz.org/2.json"], + ), ( b'\n', ["http://www.wfuzz.org/android-chrome-manifest.json"], From 13473ffb04555e4f67d4b54333748e5d3334d112 Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 1 Oct 2020 20:07:45 +0200 Subject: [PATCH 08/89] only enqueue plugin results on http transport --- src/wfuzz/fuzzqueues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index 939bab1c..f90a25d0 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -348,7 +348,7 @@ def process_results(self, res, plugins_res_queue): if Facade().sett.get("general", "cancel_on_plugin_except") == "1": self._throw(item._exception) res.plugins_res.append(item) - elif item._seed is not None: + elif item._seed is not None and self.options["transport"] == "http": cache_hit = self.cache.update_cache(item._seed.history, "backfeed") if (self.options["no_cache"] or cache_hit) and ( self.max_dlevel == 0 or self.max_dlevel >= res.rlevel @@ -357,7 +357,7 @@ def process_results(self, res, plugins_res_queue): self.stats.pending_fuzz.inc() self.send(item._seed) enq_item[item.source] += 1 - else: + elif item.issue: res.plugins_res.append(item) for plugin_name, enq_num in enq_item.items(): From a571b303116ce7834a3592efd1cd1fb6a58b801c Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 21:16:51 +0200 Subject: [PATCH 09/89] refactor header plugin --- src/wfuzz/plugins/scripts/headers.py | 47 +++++++++++++--------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index 19732412..c8c070aa 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -2,14 +2,17 @@ from wfuzz.externals.moduleman.plugin import moduleman_plugin +KBASE_KEY = "http.headers.servers" + + @moduleman_plugin class headers(BasePlugin): name = "headers" author = ("Xavi Mendez (@xmendez)",) version = "0.1" - summary = "Looks for server headers" + summary = "Looks for new headers associated to HTTP servers in HTTP responses" description = ("Looks for new server headers",) - category = ["verbose", "passive"] + category = ["info", "passive", "default"] priority = 99 parameters = () @@ -20,27 +23,19 @@ def validate(self, fuzzresult): return True def process(self, fuzzresult): - serverh = "" - poweredby = "" - - if "Server" in fuzzresult.history.headers.response: - serverh = fuzzresult.history.headers.response["Server"] - - if "X-Powered-By" in fuzzresult.history.headers.response: - poweredby = fuzzresult.history.headers.response["X-Powered-By"] - - if serverh != "": - if "server" not in self.kbase: - self.kbase["server"] = serverh - self.add_result("Server header first set - " + serverh) - elif serverh not in self.kbase["server"]: - self.kbase["server"] = serverh - self.add_result("New Server header - " + serverh) - - if poweredby != "": - if "poweredby" not in self.kbase: - self.kbase["poweredby"] = poweredby - self.add_result("Powered-by header first set - " + poweredby) - elif poweredby not in self.kbase["poweredby"]: - self.kbase["poweredby"] = poweredby - self.add_result("New X-Powered-By header - " + poweredby) + + watch_headers = [ + "Server", + "X-Powered-By" + ] + + for header in watch_headers: + header_value = None + if header in fuzzresult.history.headers.response: + header_value = fuzzresult.history.headers.response[header] + + if header_value is not None: + if header_value.lower() not in self.kbase[KBASE_KEY] or KBASE_KEY not in self.kbase: + self.add_result("New server header. {}: {}".format(header, header_value)) + + self.kbase[KBASE_KEY].append(header_value.lower()) From f39ff7b1f68e1f476817746b23b668a0f01fc902 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 21:17:20 +0200 Subject: [PATCH 10/89] use constants for kbase entries --- src/wfuzz/plugins/scripts/links.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index e645410d..efb58887 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -12,6 +12,11 @@ from wfuzz.externals.moduleman.plugin import moduleman_plugin +KBASE_PARAM_PATH = "links.add_path" +KBASE_PARAM_DOMAIN_REGEX = "links.regex" +KBASE_NEW_DOMAIN = "links.new_domains" + + @moduleman_plugin class links(BasePlugin, DiscoveryPluginMixin): name = "links" @@ -48,12 +53,12 @@ def __init__(self): ("Location", re.compile(r"(.*)")), ] - self.add_path = self.kbase["links.add_path"] + self.add_path = self.kbase[KBASE_PARAM_PATH] self.domain_regex = None - if self.kbase["links.regex"][0]: + if self.kbase[KBASE_PARAM_DOMAIN_REGEX][0]: self.domain_regex = re.compile( - self.kbase["links.regex"][0], re.MULTILINE | re.DOTALL + self.kbase[KBASE_PARAM_DOMAIN_REGEX][0], re.MULTILINE | re.DOTALL ) self.list_links = set() @@ -119,9 +124,9 @@ def from_domain(self, fuzzresult, parsed_link): if ( parsed_link.netloc - and parsed_link.netloc not in self.kbase["links.new_domains"] + and parsed_link.netloc not in self.kbase[KBASE_NEW_DOMAIN] ): - self.kbase["links.new_domains"].append(parsed_link.netloc) + self.kbase[KBASE_NEW_DOMAIN].append(parsed_link.netloc) self.add_result( "New domain found, link not enqueued %s" % parsed_link.netloc ) From c1e1e8b1056ec00d78e6cb18b90aebc1fe275e3f Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 21:20:32 +0200 Subject: [PATCH 11/89] change A,AA,AAA categories --- src/wfuzz/ui/console/clparser.py | 2 +- src/wfuzz/ui/console/common.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/ui/console/clparser.py b/src/wfuzz/ui/console/clparser.py index 3a87676d..7d61e74c 100644 --- a/src/wfuzz/ui/console/clparser.py +++ b/src/wfuzz/ui/console/clparser.py @@ -612,7 +612,7 @@ def _parse_scripts(self, optsd, options): options["script"] = "default,verbose" if "--AAA" in optsd: - options["script"] = "default,discovery,verbose" + options["script"] = "default,verbose,discovery" if "--script" in optsd: options["script"] = ( diff --git a/src/wfuzz/ui/console/common.py b/src/wfuzz/ui/console/common.py index ac167cea..c5d4fc94 100644 --- a/src/wfuzz/ui/console/common.py +++ b/src/wfuzz/ui/console/common.py @@ -128,7 +128,7 @@ \t--req-delay N : Sets the maximum time in seconds the request is allowed to take (CURLOPT_TIMEOUT). Default 90. \t--conn-delay N : Sets the maximum time in seconds the connection phase to the server to take (CURLOPT_CONNECTTIMEOUT). Default 90. \t -\t-A, --AA, --AAA : Alias for --script=default,verbose,discovery -v -c +\t-A, --AA, --AAA : Alias for -v -c and --script=default,verbose,discover respectively \t--no-cache : Disable plugins cache. Every request will be scanned. \t--script= : Equivalent to --script=default \t--script= : Runs script's scan. is a comma separated list of plugin-files or plugin-categories @@ -182,7 +182,7 @@ \t--efield : Show the specified language expression together with the current payload. Repeat option for various fields. \t--field : Do not show the payload but only the specified language expression. Repeat option for various fields. \t -\t-A, --AA : Alias for --script=default,verbose -v -c +\t-A, --AA, --AAA : Alias for -v -c and --script=default,verbose,discover respectively \t--script= : Equivalent to --script=default \t--script= : Runs script's scan. is a comma separated list of plugin-files or plugin-categories \t--script-help= : Show help about scripts. From 44d7fd33688565c2d961aa8651b650a5d7def3a4 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 21:21:18 +0200 Subject: [PATCH 12/89] change plugin categories --- src/wfuzz/plugins/scripts/backups.py | 2 +- src/wfuzz/plugins/scripts/cookies.py | 2 +- src/wfuzz/plugins/scripts/cvs_extractor.py | 2 +- src/wfuzz/plugins/scripts/errors.py | 2 +- src/wfuzz/plugins/scripts/npm_deps.py | 2 +- src/wfuzz/plugins/scripts/robots.py | 2 +- src/wfuzz/plugins/scripts/sitemap.py | 2 +- src/wfuzz/plugins/scripts/svn_extractor.py | 2 +- src/wfuzz/plugins/scripts/wcdb.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/wfuzz/plugins/scripts/backups.py b/src/wfuzz/plugins/scripts/backups.py index 880566a5..eb0a70a0 100644 --- a/src/wfuzz/plugins/scripts/backups.py +++ b/src/wfuzz/plugins/scripts/backups.py @@ -19,7 +19,7 @@ class backups(BasePlugin): "* http://localhost/dir.EXTENSIONS", author = ("Xavi Mendez (@xmendez)",) version = "0.1" - category = ["re-enqueue", "active", "discovery"] + category = ["fuzzer", "active"] priority = 99 parameters = ( diff --git a/src/wfuzz/plugins/scripts/cookies.py b/src/wfuzz/plugins/scripts/cookies.py index eb835929..57a767f1 100644 --- a/src/wfuzz/plugins/scripts/cookies.py +++ b/src/wfuzz/plugins/scripts/cookies.py @@ -9,7 +9,7 @@ class cookies(BasePlugin): version = "0.1" summary = "Looks for new cookies" description = ("Looks for new cookies",) - category = ["verbose", "passive"] + category = ["info", "passive" , "default"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/cvs_extractor.py b/src/wfuzz/plugins/scripts/cvs_extractor.py index 05835598..ce5c6f70 100644 --- a/src/wfuzz/plugins/scripts/cvs_extractor.py +++ b/src/wfuzz/plugins/scripts/cvs_extractor.py @@ -22,7 +22,7 @@ class cvs_extractor(BasePlugin, DiscoveryPluginMixin): version = "0.1" summary = "Parses CVS/Entries file." description = ("Parses CVS/Entries file and enqueues found entries",) - category = ["default", "active", "discovery"] + category = ["active", "discovery"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/errors.py b/src/wfuzz/plugins/scripts/errors.py index 31208f54..33a4f21a 100644 --- a/src/wfuzz/plugins/scripts/errors.py +++ b/src/wfuzz/plugins/scripts/errors.py @@ -11,7 +11,7 @@ class errors(BasePlugin): version = "0.1" summary = "Looks for error messages" description = ("Looks for common error messages",) - category = ["default", "passive"] + category = ["default", "passive", "info"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/npm_deps.py b/src/wfuzz/plugins/scripts/npm_deps.py index c54e9bf2..8fc00f4d 100644 --- a/src/wfuzz/plugins/scripts/npm_deps.py +++ b/src/wfuzz/plugins/scripts/npm_deps.py @@ -14,7 +14,7 @@ class npm_deps(BasePlugin): description = ( "Extracts npm packages by using regex pattern from the HTTP response and prints it", ) - category = ["default"] + category = ["info", "verbose"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/robots.py b/src/wfuzz/plugins/scripts/robots.py index 57851ae6..037c9aea 100644 --- a/src/wfuzz/plugins/scripts/robots.py +++ b/src/wfuzz/plugins/scripts/robots.py @@ -19,7 +19,7 @@ class robots(BasePlugin, DiscoveryPluginMixin): version = "0.1" summary = "Parses robots.txt looking for new content." description = ("Parses robots.txt looking for new content.",) - category = ["default", "active", "discovery"] + category = ["active", "discovery"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/sitemap.py b/src/wfuzz/plugins/scripts/sitemap.py index 3328e1e7..310e2da6 100644 --- a/src/wfuzz/plugins/scripts/sitemap.py +++ b/src/wfuzz/plugins/scripts/sitemap.py @@ -13,7 +13,7 @@ class sitemap(BasePlugin, DiscoveryPluginMixin): version = "0.1" summary = "Parses sitemap.xml file" description = ("Parses sitemap.xml file",) - category = ["default", "active", "discovery"] + category = ["active", "discovery"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/svn_extractor.py b/src/wfuzz/plugins/scripts/svn_extractor.py index 8e1bf468..2d4004bd 100644 --- a/src/wfuzz/plugins/scripts/svn_extractor.py +++ b/src/wfuzz/plugins/scripts/svn_extractor.py @@ -16,7 +16,7 @@ class svn_extractor(BasePlugin, DiscoveryPluginMixin): version = "0.1" summary = "Parses .svn/entries file." description = ("Parses CVS/Entries file and enqueues found entries",) - category = ["default", "active", "discovery"] + category = ["active", "discovery"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/wcdb.py b/src/wfuzz/plugins/scripts/wcdb.py index 3d128412..9e819769 100644 --- a/src/wfuzz/plugins/scripts/wcdb.py +++ b/src/wfuzz/plugins/scripts/wcdb.py @@ -20,7 +20,7 @@ class wcdb_extractor(BasePlugin, DiscoveryPluginMixin): version = "0.1" summary = "Parses subversion's wc.db file." description = ("Parses subversion's wc.db file.",) - category = ["default", "active", "discovery"] + category = ["active", "discovery"] priority = 99 parameters = () From 978a12ca46f5d1105375f031d05ee2eb9203a219 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 22:04:34 +0200 Subject: [PATCH 13/89] minor change to headers plugin --- src/wfuzz/plugins/scripts/headers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index c8c070aa..31c3f74e 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -2,7 +2,7 @@ from wfuzz.externals.moduleman.plugin import moduleman_plugin -KBASE_KEY = "http.headers.servers" +KBASE_KEY = "http.response.headers.servers" @moduleman_plugin @@ -10,7 +10,7 @@ class headers(BasePlugin): name = "headers" author = ("Xavi Mendez (@xmendez)",) version = "0.1" - summary = "Looks for new headers associated to HTTP servers in HTTP responses" + summary = "Looks for new response headers associated to HTTP servers in HTTP responses" description = ("Looks for new server headers",) category = ["info", "passive", "default"] priority = 99 @@ -19,23 +19,24 @@ class headers(BasePlugin): def __init__(self): BasePlugin.__init__(self) + self.watch_headers = [ + "Server", + "X-Powered-By" + "Via" + ] + def validate(self, fuzzresult): return True def process(self, fuzzresult): - watch_headers = [ - "Server", - "X-Powered-By" - ] - - for header in watch_headers: + for header in self.watch_headers: header_value = None if header in fuzzresult.history.headers.response: header_value = fuzzresult.history.headers.response[header] if header_value is not None: if header_value.lower() not in self.kbase[KBASE_KEY] or KBASE_KEY not in self.kbase: - self.add_result("New server header. {}: {}".format(header, header_value)) + self.add_result("New server response header. {}: {}".format(header, header_value)) self.kbase[KBASE_KEY].append(header_value.lower()) From aaddcbab75d4ac71b87419fbf3357c57ddbb1202 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 22:07:05 +0200 Subject: [PATCH 14/89] new uncommon headers plugins --- src/wfuzz/plugins/scripts/uncommon_headers.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/wfuzz/plugins/scripts/uncommon_headers.py diff --git a/src/wfuzz/plugins/scripts/uncommon_headers.py b/src/wfuzz/plugins/scripts/uncommon_headers.py new file mode 100644 index 00000000..3068a17a --- /dev/null +++ b/src/wfuzz/plugins/scripts/uncommon_headers.py @@ -0,0 +1,84 @@ +from wfuzz.plugin_api.base import BasePlugin +from wfuzz.externals.moduleman.plugin import moduleman_plugin + +import re + + +KBASE_KEY_UNCOMMON = "http.response.headers.uncommon" + + +@moduleman_plugin +class unheaders(BasePlugin): + name = "unheaders" + author = ("Xavi Mendez (@xmendez)",) + version = "0.1" + summary = "Looks for new uncommon reponse headers" + description = ("Looks for new uncommon http reponse headers",) + category = ["info", "passive", "verbose"] + priority = 99 + parameters = () + + def __init__(self): + BasePlugin.__init__(self) + + common_headers_regex = [ + r"^Server$", + r"^X-Powered-By$", + r"^Via$", + + r"^Access-Control.*$", + r"^Accept-.*$", + r"^age$", + r"^allow$", + r"^Cache-control$", + r"^Client-.*$", + r"^Connection$", + r"^Content-.*$", + r"^Date$", + r"^Etag$", + r"^Expires$", + r"^Keep-Alive$", + r"^Last-Modified$", + r"^Link$", + r"^Location$", + r"^P3P$", + r"^Pragma$", + r"^Proxy-.*$", + r"^Refresh$", + r"^Retry-After$", + r"^Referrer-Policy$", + r"^Set-Cookie$", + r"^Server-Timing$", + r"^Status$", + r"^Strict-Transport-Security$", + r"^Timing-Allow-Origin$", + r"^Trailer$", + r"^Transfer-Encoding$", + r"^Upgrade$", + r"^Vary$", + r"^Warning^$", + r"^WWW-Authenticate$", + r"^X-Content-Type-Options$", + r"^X-Download-Options$", + r"^X-Frame-Options$", + r"^X-Microsite$", + r"^X-Request-Handler-Origin-Region$", + r"^X-XSS-Protection$", + ] + + self.common_headers_regex = re.compile("({})".format("|".join(common_headers_regex)), re.IGNORECASE) + + def validate(self, fuzzresult): + return True + + def process(self, fuzzresult): + for header in fuzzresult.history.headers.response.keys(): + header_value = None + if not self.common_headers_regex.match(header): + header_value = header + + if header_value is not None: + if header_value.lower() not in self.kbase[KBASE_KEY_UNCOMMON] or KBASE_KEY_UNCOMMON not in self.kbase: + self.add_result("New uncommon reponse header. {}: {}".format(header_value, fuzzresult.history.headers.response[header_value])) + + self.kbase[KBASE_KEY_UNCOMMON].append(header_value.lower()) From dc2f41b628967094f136d945e68292edd54d52f4 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 22:11:56 +0200 Subject: [PATCH 15/89] change npm deps category --- src/wfuzz/plugins/scripts/npm_deps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/scripts/npm_deps.py b/src/wfuzz/plugins/scripts/npm_deps.py index 8fc00f4d..1e73c0f7 100644 --- a/src/wfuzz/plugins/scripts/npm_deps.py +++ b/src/wfuzz/plugins/scripts/npm_deps.py @@ -14,7 +14,7 @@ class npm_deps(BasePlugin): description = ( "Extracts npm packages by using regex pattern from the HTTP response and prints it", ) - category = ["info", "verbose"] + category = ["info"] priority = 99 parameters = () From 7d6725b79a2a1b320412bbf490444140337666a2 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 2 Oct 2020 22:13:09 +0200 Subject: [PATCH 16/89] change title category --- src/wfuzz/plugins/scripts/title.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/scripts/title.py b/src/wfuzz/plugins/scripts/title.py index 36552854..aeff0fff 100644 --- a/src/wfuzz/plugins/scripts/title.py +++ b/src/wfuzz/plugins/scripts/title.py @@ -9,7 +9,7 @@ class title(BasePlugin): version = "0.1" summary = "Parses HTML page title" description = ("Parses HTML page title",) - category = ["verbose", "passive"] + category = ["info", "passive"] priority = 99 parameters = () From a8f08a49c0a981056f3459c6289523b12ba19960 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 6 Oct 2020 19:37:04 +0200 Subject: [PATCH 17/89] increase words width --- src/wfuzz/ui/console/mvc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 6c77f2c1..ce095b83 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -155,8 +155,8 @@ def on_stats(self, **event): class View: - widths = [10, 8, 6, 6, 9, getTerminalSize()[0] - 65] - verbose_widths = [10, 10, 8, 6, 6, 9, 30, 30, getTerminalSize()[0] - 145] + widths = [10, 8, 6, 8, 9, getTerminalSize()[0] - 65] + verbose_widths = [10, 10, 8, 8, 6, 9, 30, 30, getTerminalSize()[0] - 145] def __init__(self, session_options): self.colour = session_options["colour"] From eba16d7be759cf84393186cf2c1322a5ee8606fb Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 6 Oct 2020 19:42:41 +0200 Subject: [PATCH 18/89] add header --- src/wfuzz/plugins/scripts/uncommon_headers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wfuzz/plugins/scripts/uncommon_headers.py b/src/wfuzz/plugins/scripts/uncommon_headers.py index 3068a17a..1cae5ec8 100644 --- a/src/wfuzz/plugins/scripts/uncommon_headers.py +++ b/src/wfuzz/plugins/scripts/uncommon_headers.py @@ -34,6 +34,7 @@ def __init__(self): r"^Client-.*$", r"^Connection$", r"^Content-.*$", + r"^Cross-Origin-Resource-Policy$", r"^Date$", r"^Etag$", r"^Expires$", From d37d022eaf29811700aa863b748baafe979223cb Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 8 Oct 2020 19:23:01 +0200 Subject: [PATCH 19/89] black formatting --- src/wfuzz/plugins/scripts/cookies.py | 2 +- src/wfuzz/plugins/scripts/headers.py | 21 ++++++++++++------- src/wfuzz/plugins/scripts/uncommon_headers.py | 17 +++++++++++---- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/wfuzz/plugins/scripts/cookies.py b/src/wfuzz/plugins/scripts/cookies.py index 57a767f1..210b3ee4 100644 --- a/src/wfuzz/plugins/scripts/cookies.py +++ b/src/wfuzz/plugins/scripts/cookies.py @@ -9,7 +9,7 @@ class cookies(BasePlugin): version = "0.1" summary = "Looks for new cookies" description = ("Looks for new cookies",) - category = ["info", "passive" , "default"] + category = ["info", "passive", "default"] priority = 99 parameters = () diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index 31c3f74e..b74707a9 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -10,7 +10,9 @@ class headers(BasePlugin): name = "headers" author = ("Xavi Mendez (@xmendez)",) version = "0.1" - summary = "Looks for new response headers associated to HTTP servers in HTTP responses" + summary = ( + "Looks for new response headers associated to HTTP servers in HTTP responses" + ) description = ("Looks for new server headers",) category = ["info", "passive", "default"] priority = 99 @@ -19,11 +21,7 @@ class headers(BasePlugin): def __init__(self): BasePlugin.__init__(self) - self.watch_headers = [ - "Server", - "X-Powered-By" - "Via" - ] + self.watch_headers = ["Server", "X-Powered-By" "Via"] def validate(self, fuzzresult): return True @@ -36,7 +34,14 @@ def process(self, fuzzresult): header_value = fuzzresult.history.headers.response[header] if header_value is not None: - if header_value.lower() not in self.kbase[KBASE_KEY] or KBASE_KEY not in self.kbase: - self.add_result("New server response header. {}: {}".format(header, header_value)) + if ( + header_value.lower() not in self.kbase[KBASE_KEY] + or KBASE_KEY not in self.kbase + ): + self.add_result( + "New server response header. {}: {}".format( + header, header_value + ) + ) self.kbase[KBASE_KEY].append(header_value.lower()) diff --git a/src/wfuzz/plugins/scripts/uncommon_headers.py b/src/wfuzz/plugins/scripts/uncommon_headers.py index 1cae5ec8..d0e1a730 100644 --- a/src/wfuzz/plugins/scripts/uncommon_headers.py +++ b/src/wfuzz/plugins/scripts/uncommon_headers.py @@ -25,7 +25,6 @@ def __init__(self): r"^Server$", r"^X-Powered-By$", r"^Via$", - r"^Access-Control.*$", r"^Accept-.*$", r"^age$", @@ -67,7 +66,9 @@ def __init__(self): r"^X-XSS-Protection$", ] - self.common_headers_regex = re.compile("({})".format("|".join(common_headers_regex)), re.IGNORECASE) + self.common_headers_regex = re.compile( + "({})".format("|".join(common_headers_regex)), re.IGNORECASE + ) def validate(self, fuzzresult): return True @@ -79,7 +80,15 @@ def process(self, fuzzresult): header_value = header if header_value is not None: - if header_value.lower() not in self.kbase[KBASE_KEY_UNCOMMON] or KBASE_KEY_UNCOMMON not in self.kbase: - self.add_result("New uncommon reponse header. {}: {}".format(header_value, fuzzresult.history.headers.response[header_value])) + if ( + header_value.lower() not in self.kbase[KBASE_KEY_UNCOMMON] + or KBASE_KEY_UNCOMMON not in self.kbase + ): + self.add_result( + "New uncommon reponse header. {}: {}".format( + header_value, + fuzzresult.history.headers.response[header_value], + ) + ) self.kbase[KBASE_KEY_UNCOMMON].append(header_value.lower()) From bda110d31d5e09848c4ff94488f60aa172f507e6 Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 9 Oct 2020 00:00:53 +0200 Subject: [PATCH 20/89] plugin split data and msg --- src/wfuzz/factories/plugin_factory.py | 4 +++- src/wfuzz/fuzzobjects.py | 2 ++ src/wfuzz/fuzzqueues.py | 8 ++++++-- src/wfuzz/plugin_api/base.py | 4 ++-- src/wfuzz/plugins/scripts/cookies.py | 13 +++++++++---- src/wfuzz/plugins/scripts/errors.py | 2 +- src/wfuzz/plugins/scripts/grep.py | 2 +- src/wfuzz/plugins/scripts/headers.py | 6 +++--- src/wfuzz/plugins/scripts/links.py | 2 +- src/wfuzz/plugins/scripts/listing.py | 2 +- src/wfuzz/plugins/scripts/npm_deps.py | 4 ++-- src/wfuzz/plugins/scripts/screenshot.py | 2 +- src/wfuzz/plugins/scripts/svn_extractor.py | 2 +- src/wfuzz/plugins/scripts/title.py | 2 +- src/wfuzz/plugins/scripts/uncommon_headers.py | 6 ++++-- src/wfuzz/plugins/scripts/wcdb.py | 4 ++-- src/wfuzz/ui/console/mvc.py | 3 +++ 17 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/wfuzz/factories/plugin_factory.py b/src/wfuzz/factories/plugin_factory.py index 4a558341..f6d66a9f 100644 --- a/src/wfuzz/factories/plugin_factory.py +++ b/src/wfuzz/factories/plugin_factory.py @@ -38,10 +38,12 @@ def __call__(self, name, exception): class PluginFindingBuilder: - def __call__(self, name, message): + def __call__(self, name, itype, message, data): plugin = FuzzPlugin() plugin.source = name plugin.issue = message + plugin.itype = itype + plugin.data = data plugin._exception = None plugin._seed = None diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 74440077..7f5bb2de 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -401,5 +401,7 @@ def __init__(self): FuzzItem.__init__(self, FuzzType.PLUGIN) self.source = "" self.issue = "" + self.itype = "" + self.data = "" self._exception = None self._seed = None diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index f90a25d0..f6fb5222 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -364,9 +364,11 @@ def process_results(self, res, plugins_res_queue): res.plugins_res.append( plugin_factory.create( "plugin_from_finding", - "Backfeed", + "output", + "msg", "Plugin %s enqueued %d more requests (rlevel=%d)" % (plugin_name, enq_num, res.rlevel), + None, ) ) @@ -392,8 +394,10 @@ def process(self, fuzz_res): fuzz_res.plugins_res.append( plugin_factory.create( "plugin_from_finding", - "Recursion", + "output", + "msg", "Enqueued response for recursion (level=%d)" % (seed.rlevel), + None, ) ) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index 80cfdbae..c85ea69f 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -58,9 +58,9 @@ def process(self, fuzzresult): def validate(self): raise FuzzExceptPluginError("Method count not implemented") - def add_result(self, issue): + def add_result(self, itype, issue, data): self.results_queue.put( - plugin_factory.create("plugin_from_finding", self.name, issue) + plugin_factory.create("plugin_from_finding", self.name, itype, issue, data) ) def queue_url(self, url): diff --git a/src/wfuzz/plugins/scripts/cookies.py b/src/wfuzz/plugins/scripts/cookies.py index 210b3ee4..ed31ec25 100644 --- a/src/wfuzz/plugins/scripts/cookies.py +++ b/src/wfuzz/plugins/scripts/cookies.py @@ -2,6 +2,9 @@ from wfuzz.externals.moduleman.plugin import moduleman_plugin +KBASE_NEW_COOKIE = "cookies.cookie" + + @moduleman_plugin class cookies(BasePlugin): name = "cookies" @@ -28,8 +31,10 @@ def process(self, fuzzresult): if ( name != "" - and "cookie" not in self.kbase - or name not in self.kbase["cookie"] + and KBASE_NEW_COOKIE not in self.kbase + or name not in self.kbase[KBASE_NEW_COOKIE] ): - self.kbase["cookie"] = name - self.add_result("Cookie first set - %s=%s" % (name, value)) + self.kbase[KBASE_NEW_COOKIE] = name + self.add_result( + "cookie", "Cookie first set", "%s=%s" % (name, value) + ) diff --git a/src/wfuzz/plugins/scripts/errors.py b/src/wfuzz/plugins/scripts/errors.py index 33a4f21a..e64e3f4e 100644 --- a/src/wfuzz/plugins/scripts/errors.py +++ b/src/wfuzz/plugins/scripts/errors.py @@ -111,4 +111,4 @@ def validate(self, fuzzresult): def process(self, fuzzresult): for regex in self.error_regex: for regex_match in regex.findall(fuzzresult.history.content): - self.add_result("Error identified: {}".format(regex_match)) + self.add_result("errors", "Error identified", regex_match) diff --git a/src/wfuzz/plugins/scripts/grep.py b/src/wfuzz/plugins/scripts/grep.py index df6ff295..79e7f160 100644 --- a/src/wfuzz/plugins/scripts/grep.py +++ b/src/wfuzz/plugins/scripts/grep.py @@ -37,4 +37,4 @@ def validate(self, fuzzresult): def process(self, fuzzresult): for r in self.regex.findall(fuzzresult.history.content): - self.add_result("Pattern match %s" % r) + self.add_result("match", "Pattern match", r) diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index b74707a9..e12dfbd4 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -39,9 +39,9 @@ def process(self, fuzzresult): or KBASE_KEY not in self.kbase ): self.add_result( - "New server response header. {}: {}".format( - header, header_value - ) + "header", + "New server response header", + "{}: {}".format(header, header_value), ) self.kbase[KBASE_KEY].append(header_value.lower()) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index efb58887..8bddafcf 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -128,5 +128,5 @@ def from_domain(self, fuzzresult, parsed_link): ): self.kbase[KBASE_NEW_DOMAIN].append(parsed_link.netloc) self.add_result( - "New domain found, link not enqueued %s" % parsed_link.netloc + "domain", "New domain found, link not enqueued", parsed_link.netloc ) diff --git a/src/wfuzz/plugins/scripts/listing.py b/src/wfuzz/plugins/scripts/listing.py index d49b5247..b5f114ce 100644 --- a/src/wfuzz/plugins/scripts/listing.py +++ b/src/wfuzz/plugins/scripts/listing.py @@ -47,5 +47,5 @@ def validate(self, fuzzresult): def process(self, fuzzresult): for r in self.regex: if len(r.findall(fuzzresult.history.content)) > 0: - self.add_result("Directory listing identified") + self.add_result("msg", "Directory listing identified", None) break diff --git a/src/wfuzz/plugins/scripts/npm_deps.py b/src/wfuzz/plugins/scripts/npm_deps.py index 1e73c0f7..dde3156a 100644 --- a/src/wfuzz/plugins/scripts/npm_deps.py +++ b/src/wfuzz/plugins/scripts/npm_deps.py @@ -42,8 +42,8 @@ def validate(self, fuzzresult): def process(self, fuzzresult): if self.match_dev: for name, version in self.REGEX_PATT.findall(self.match_dev.group(1)): - self.add_result(name) + self.add_result("dependency", "npm dependency", name) if self.match: for name, version in self.REGEX_PATT.findall(self.match.group(1)): - self.add_result(name) + self.add_result("dev_dependency", "npm dev dependency", name) diff --git a/src/wfuzz/plugins/scripts/screenshot.py b/src/wfuzz/plugins/scripts/screenshot.py index 0e658856..4247d896 100644 --- a/src/wfuzz/plugins/scripts/screenshot.py +++ b/src/wfuzz/plugins/scripts/screenshot.py @@ -41,4 +41,4 @@ def process(self, fuzzresult): "--out=%s" % filename, ] ) - self.add_result("Screnshot taken, output at %s" % filename) + self.add_result("file", "Screnshot taken", filename) diff --git a/src/wfuzz/plugins/scripts/svn_extractor.py b/src/wfuzz/plugins/scripts/svn_extractor.py index 2d4004bd..724bfb1f 100644 --- a/src/wfuzz/plugins/scripts/svn_extractor.py +++ b/src/wfuzz/plugins/scripts/svn_extractor.py @@ -57,7 +57,7 @@ def process(self, fuzzresult): file_list, dir_list, author_list = self.readsvn(fuzzresult.history.content) if author_list: - self.add_result("SVN authors: %s" % ", ".join(author_list)) + self.add_result("authors", "SVN authors", ", ".join(author_list)) for f in file_list: u = urljoin(base_url.replace("/.svn/", "/"), f) diff --git a/src/wfuzz/plugins/scripts/title.py b/src/wfuzz/plugins/scripts/title.py index aeff0fff..dd756e37 100644 --- a/src/wfuzz/plugins/scripts/title.py +++ b/src/wfuzz/plugins/scripts/title.py @@ -30,4 +30,4 @@ def process(self, fuzzresult): or title not in self.kbase["title"] ): self.kbase["title"] = title - self.add_result("Page title: %s" % title) + self.add_result("title", "Page title", title) diff --git a/src/wfuzz/plugins/scripts/uncommon_headers.py b/src/wfuzz/plugins/scripts/uncommon_headers.py index d0e1a730..0778134f 100644 --- a/src/wfuzz/plugins/scripts/uncommon_headers.py +++ b/src/wfuzz/plugins/scripts/uncommon_headers.py @@ -85,10 +85,12 @@ def process(self, fuzzresult): or KBASE_KEY_UNCOMMON not in self.kbase ): self.add_result( - "New uncommon reponse header. {}: {}".format( + "header", + "New uncommon reponse header", + "{}: {}".format( header_value, fuzzresult.history.headers.response[header_value], - ) + ), ) self.kbase[KBASE_KEY_UNCOMMON].append(header_value.lower()) diff --git a/src/wfuzz/plugins/scripts/wcdb.py b/src/wfuzz/plugins/scripts/wcdb.py index 9e819769..2308f1a1 100644 --- a/src/wfuzz/plugins/scripts/wcdb.py +++ b/src/wfuzz/plugins/scripts/wcdb.py @@ -65,10 +65,10 @@ def process(self, fuzzresult): author_list, list_items = self.readwc(fuzzresult.history.content) if author_list: - self.add_result("SVN authors: %s" % ", ".join(author_list)) + self.add_result("authors", "SVN authors", ", ".join(author_list)) if list_items: for f, pristine in list_items: u = urljoin(fuzzresult.url.replace("/.svn/wc.db", "/"), f) if self.queue_url(u): - self.add_result("SVN %s source code in %s" % (f, pristine)) + self.add_result("source", "SVN source code", f) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index ce095b83..bd11a668 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -325,6 +325,9 @@ def result(self, res): if res.plugins_res: for i in res.plugins_res: + sys.stdout.write( + " |_ {} {}\r".format(i.issue, i.data if i.data else "") + ) sys.stdout.write(" |_ %s\r" % i.issue) sys.stdout.write("\n\r") From cb3612d21dced2d149e1aff2ed3db360363c790d Mon Sep 17 00:00:00 2001 From: javi Date: Fri, 9 Oct 2020 01:03:25 +0200 Subject: [PATCH 21/89] plugins field as dotdict --- src/wfuzz/filters/ppfilter.py | 27 ++++++++++++++------------- src/wfuzz/fuzzobjects.py | 18 ++++++++++++++---- src/wfuzz/helpers/obj_dic.py | 9 +++++++++ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/wfuzz/filters/ppfilter.py b/src/wfuzz/filters/ppfilter.py index 50b76704..2eef5bfe 100644 --- a/src/wfuzz/filters/ppfilter.py +++ b/src/wfuzz/filters/ppfilter.py @@ -58,7 +58,7 @@ def __init__(self, filter_string=None): r"FUZ(?P\d)*Z(?:\[(?P(\w|_|-|\.)+)\])?", asMatch=True ).setParseAction(self._compute_fuzz_symbol) res_symbol = Regex( - r"(description|nres|code|chars|lines|words|md5|content|timer|url|plugins|l|w|c|(r|history)(\w|_|-|\.)*|h)" + r"(description|nres|code|chars|lines|words|md5|content|timer|url|l|w|c|(r|history|plugins)(\w|_|-|\.)*|h)" ).setParseAction(self._compute_res_symbol) bbb_symbol = Regex( r"BBB(?:\[(?P(\w|_|-|\.)+)\])?", asMatch=True @@ -132,6 +132,8 @@ def __compute_res_value(self, tokens): location, fuzz_val, operator_match.groupdict() ) + if isinstance(fuzz_val, list): + return [fuzz_val] return fuzz_val def _get_payload_value(self, p_index): @@ -146,7 +148,7 @@ def _get_field_value(self, fuzz_val, field): self.stack.append(field) try: - return rgetattr(fuzz_val, field) + ret = rgetattr(fuzz_val, field) except IndexError: raise FuzzExceptIncorrectFilter( "Non existent FUZZ payload! Use a correct index." @@ -158,6 +160,10 @@ def _get_field_value(self, fuzz_val, field): ) ) + if isinstance(ret, list): + return [ret] + return ret + def __compute_bbb_symbol(self, tokens): if self.baseline is None: raise FuzzExceptBadOptions( @@ -274,17 +280,7 @@ def __compute_expr(self, tokens): elif isinstance(leftvalue, list): ret = value_in_any_list_item(rightvalue, leftvalue) elif isinstance(leftvalue, dict) or isinstance(leftvalue, DotDict): - return ( - len( - { - k: v - for (k, v) in leftvalue.items() - if rightvalue.lower() in k.lower() - or value_in_any_list_item(rightvalue, v) - } - ) - > 0 - ) + ret = rightvalue.lower() in str(leftvalue) else: raise FuzzExceptBadOptions( "Invalid operand type {}".format(rightvalue) @@ -322,6 +318,9 @@ def __myreduce(self, elements): first = first or elements[i + 1] self.stack = [] + + if isinstance(first, list): + return [first] return first def __compute_not_operator(self, tokens): @@ -330,6 +329,8 @@ def __compute_not_operator(self, tokens): if operator == "not": return not value + if isinstance(value, list): + return [value] return value def __compute_formula(self, tokens): diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 7f5bb2de..291e871c 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -13,6 +13,7 @@ from .helpers.str_func import python2_3_convert_to_unicode from .helpers.obj_dyn import rgetattr from .helpers.utils import MyCounter +from .helpers.obj_dic import DotDict FuzzWord = namedtuple("FuzzWord", ["content", "type"]) @@ -299,12 +300,18 @@ def __init__(self, history=None, exception=None, track_id=True): @property def plugins(self): - dic = defaultdict(list) + dic = defaultdict(lambda: defaultdict(list)) for pl in self.plugins_res: - dic[pl.source].append(pl.issue) + dic[pl.source][pl.itype].append(pl.data) - return dic + ret = DotDict() + for key, first in dic.items(): + ret[key] = DotDict() + for seckey, second in first.items(): + ret[key][seckey] = second + + return ret def update(self, exception=None): self.item_type = FuzzType.RESULT @@ -365,7 +372,10 @@ def eval(self, expr): return self.FUZZRESULT_SHARED_FILTER.is_visible(self, expr) def _field(self): - return " | ".join([str(self.eval(field)) for field in self._fields]) + list_eval = [self.eval(field) for field in self._fields] + return " | ".join( + ["\n".join(el) if isinstance(el, list) else str(el) for el in list_eval] + ) # parameters in common with fuzzrequest @property diff --git a/src/wfuzz/helpers/obj_dic.py b/src/wfuzz/helpers/obj_dic.py index 75c3bde0..68e91bcc 100644 --- a/src/wfuzz/helpers/obj_dic.py +++ b/src/wfuzz/helpers/obj_dic.py @@ -1,4 +1,5 @@ from collections.abc import MutableMapping +from itertools import chain class CaseInsensitiveDict(MutableMapping): @@ -65,3 +66,11 @@ def __getitem__(self, key): return super(DotDict, self).__getitem__(key) except KeyError: return DotDict({}) + + def __str__(self): + return "\n".join( + [ + "{}{} {}".format(k, "->" if isinstance(v, DotDict) else ":", v) + for k, v in self.items() + ] + ) From 1afe2ee73fce2f6425f59c2dfe0068f1ae7df716 Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 10:56:54 +0200 Subject: [PATCH 22/89] return url instead of default --- src/wfuzz/fuzzobjects.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 291e871c..f49a10ec 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -178,15 +178,16 @@ def value(self): else str(rgetattr(self.content, self.field)) ) - def description(self, default): + def description(self): if self.is_baseline: return self.content if self.marker is None: return "" + # return default value if self.field is None and isinstance(self.content, FuzzResult): - return rgetattr(self.content, default) + return self.content.url elif self.field is not None and isinstance(self.content, FuzzResult): return str(rgetattr(self.content, self.field)) @@ -253,9 +254,7 @@ def get_payloads(self): yield elem def description(self): - payl_descriptions = [ - payload.description("url") for payload in self.get_payloads() - ] + payl_descriptions = [payload.description() for payload in self.get_payloads()] ret_str = " - ".join([p_des for p_des in payl_descriptions if p_des]) return ret_str From dedbd10636fcf6f755b3cae27f0085d23def8aa7 Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 11:07:36 +0200 Subject: [PATCH 23/89] fix description when no marker --- src/wfuzz/fuzzobjects.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index f49a10ec..19efcf5f 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -344,10 +344,12 @@ def __str__(self): @property def description(self): - res_description = ( - self.payload_man.description() if self.payload_man else self.url - ) - ret_str = "" + res_description = self.payload_man.description() if self.payload_man else None + + if not res_description: + res_description = self.url + + ret_str = None if self._show_field is True: ret_str = self._field() @@ -356,9 +358,6 @@ def description(self): else: ret_str = res_description - if not ret_str: - ret_str = self.url - if self.exception: return ret_str + "! " + str(self.exception) From 7ff658acf2be97b6eb702e4dbba5b2a7b94f2a5f Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 14:07:30 +0200 Subject: [PATCH 24/89] description for list with diff separator --- src/wfuzz/fuzzobjects.py | 4 ++-- src/wfuzz/wfuzz.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 19efcf5f..0d3fdd2e 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -369,10 +369,10 @@ def description(self): def eval(self, expr): return self.FUZZRESULT_SHARED_FILTER.is_visible(self, expr) - def _field(self): + def _field(self, separator=', '): list_eval = [self.eval(field) for field in self._fields] return " | ".join( - ["\n".join(el) if isinstance(el, list) else str(el) for el in list_eval] + [separator.join(el) if isinstance(el, list) else str(el) for el in list_eval] ) # parameters in common with fuzzrequest diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index 1421fbb2..fe84dc6a 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -128,7 +128,7 @@ def usage(): if payload_type == FuzzWordType.WORD: print(res.description) elif payload_type == FuzzWordType.FUZZRES and session_options["show_field"]: - print(res._field()) + print(res._field("\n")) except KeyboardInterrupt: pass From 3d9f47c8d4451b565f86c93ab1991160f8273e85 Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 17:51:19 +0200 Subject: [PATCH 25/89] verbose msgs from plugins --- src/wfuzz/factories/plugin_factory.py | 3 ++- src/wfuzz/fuzzobjects.py | 8 ++++++-- src/wfuzz/fuzzqueues.py | 29 ++++++++++++++++----------- src/wfuzz/plugin_api/base.py | 11 +++++++++- src/wfuzz/ui/console/mvc.py | 11 +++++++--- 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/wfuzz/factories/plugin_factory.py b/src/wfuzz/factories/plugin_factory.py index f6d66a9f..e8044a07 100644 --- a/src/wfuzz/factories/plugin_factory.py +++ b/src/wfuzz/factories/plugin_factory.py @@ -38,7 +38,7 @@ def __call__(self, name, exception): class PluginFindingBuilder: - def __call__(self, name, itype, message, data): + def __call__(self, name, itype, message, data, verbose): plugin = FuzzPlugin() plugin.source = name plugin.issue = message @@ -46,6 +46,7 @@ def __call__(self, name, itype, message, data): plugin.data = data plugin._exception = None plugin._seed = None + plugin._verbose = verbose return plugin diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 0d3fdd2e..0ebaf48d 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -369,10 +369,13 @@ def description(self): def eval(self, expr): return self.FUZZRESULT_SHARED_FILTER.is_visible(self, expr) - def _field(self, separator=', '): + def _field(self, separator=", "): list_eval = [self.eval(field) for field in self._fields] return " | ".join( - [separator.join(el) if isinstance(el, list) else str(el) for el in list_eval] + [ + separator.join(el) if isinstance(el, list) else str(el) + for el in list_eval + ] ) # parameters in common with fuzzrequest @@ -413,3 +416,4 @@ def __init__(self): self.data = "" self._exception = None self._seed = None + self._verbose = False diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index f6fb5222..a4bab42a 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -339,7 +339,7 @@ def process(self, res): self.send(res) def process_results(self, res, plugins_res_queue): - enq_item = defaultdict(int) + enq_item = defaultdict(lambda: defaultdict(int)) while not plugins_res_queue.empty(): item = plugins_res_queue.get() @@ -356,21 +356,25 @@ def process_results(self, res, plugins_res_queue): self.stats.backfeed.inc() self.stats.pending_fuzz.inc() self.send(item._seed) - enq_item[item.source] += 1 + enq_item[item.source]["request"] += 1 elif item.issue: + enq_item[item.source][item.itype] += 1 res.plugins_res.append(item) - for plugin_name, enq_num in enq_item.items(): - res.plugins_res.append( - plugin_factory.create( - "plugin_from_finding", - "output", - "msg", - "Plugin %s enqueued %d more requests (rlevel=%d)" - % (plugin_name, enq_num, res.rlevel), - None, + for plugin_name, plugin_type in enq_item.items(): + for domain, enq_num in plugin_type.items(): + res.plugins_res.append( + plugin_factory.create( + "plugin_from_finding", + "output", + "msg", + "Plugin {}: {} new {}(s) added.".format( + plugin_name, enq_num, domain + ), + None, + False, + ) ) - ) class RecursiveQ(FuzzQueue): @@ -398,6 +402,7 @@ def process(self, fuzz_res): "msg", "Enqueued response for recursion (level=%d)" % (seed.rlevel), None, + False, ) ) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index c85ea69f..feca4fea 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -60,7 +60,16 @@ def validate(self): def add_result(self, itype, issue, data): self.results_queue.put( - plugin_factory.create("plugin_from_finding", self.name, itype, issue, data) + plugin_factory.create( + "plugin_from_finding", self.name, itype, issue, data, False + ) + ) + + def add_verbose_result(self, itype, issue, data): + self.results_queue.put( + plugin_factory.create( + "plugin_from_finding", self.name, itype, issue, data, True + ) ) def queue_url(self, url): diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index bd11a668..28340fbd 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -324,11 +324,16 @@ def result(self, res): self._print(prev_res, print_nres=False) if res.plugins_res: - for i in res.plugins_res: + for plugin_res in res.plugins_res: + if plugin_res._verbose and not self.verbose: + continue + sys.stdout.write( - " |_ {} {}\r".format(i.issue, i.data if i.data else "") + " |_ {} {}\r".format( + plugin_res.issue, plugin_res.data if plugin_res.data else "" + ) ) - sys.stdout.write(" |_ %s\r" % i.issue) + sys.stdout.write(" |_ %s\r" % plugin_res.issue) sys.stdout.write("\n\r") self.printed_lines = 0 From a3c3ccbbe0d8a4798c735c2dfa210193e79eb2ab Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 17:54:51 +0200 Subject: [PATCH 26/89] do not include output in plugins --- src/wfuzz/fuzzobjects.py | 4 ++++ src/wfuzz/fuzzqueues.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 0ebaf48d..1c14b0e3 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -302,6 +302,8 @@ def plugins(self): dic = defaultdict(lambda: defaultdict(list)) for pl in self.plugins_res: + if pl.source == FuzzPlugin.OUTPUT_CAT: + continue dic[pl.source][pl.itype].append(pl.data) ret = DotDict() @@ -408,6 +410,8 @@ def update_from_options(self, options): class FuzzPlugin(FuzzItem): + OUTPUT_CAT = "output" + def __init__(self): FuzzItem.__init__(self, FuzzType.PLUGIN) self.source = "" diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index a4bab42a..d0308d5c 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -8,7 +8,7 @@ from .factories.fuzzresfactory import resfactory from .factories.plugin_factory import plugin_factory from .factories.payman import payman_factory -from .fuzzobjects import FuzzType, FuzzItem, FuzzWord, FuzzWordType +from .fuzzobjects import FuzzPlugin, FuzzType, FuzzItem, FuzzWord, FuzzWordType from .myqueues import FuzzQueue from .exception import ( FuzzExceptInternalError, @@ -366,7 +366,7 @@ def process_results(self, res, plugins_res_queue): res.plugins_res.append( plugin_factory.create( "plugin_from_finding", - "output", + FuzzPlugin.OUTPUT_CAT, "msg", "Plugin {}: {} new {}(s) added.".format( plugin_name, enq_num, domain @@ -398,7 +398,7 @@ def process(self, fuzz_res): fuzz_res.plugins_res.append( plugin_factory.create( "plugin_from_finding", - "output", + FuzzPlugin.OUTPUT_CAT, "msg", "Enqueued response for recursion (level=%d)" % (seed.rlevel), None, From 47423ab373b5e46d8141acc4dddea5ddbdd19017 Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 18:05:39 +0200 Subject: [PATCH 27/89] found instead of added --- src/wfuzz/fuzzqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index d0308d5c..b9c881a8 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -368,7 +368,7 @@ def process_results(self, res, plugins_res_queue): "plugin_from_finding", FuzzPlugin.OUTPUT_CAT, "msg", - "Plugin {}: {} new {}(s) added.".format( + "Plugin {}: {} new {}(s) found.".format( plugin_name, enq_num, domain ), None, From f28881797dd791cb2e5d637f5adedbe2fd7c20aa Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 20:36:29 +0200 Subject: [PATCH 28/89] regex and new msg links --- src/wfuzz/plugins/scripts/links.py | 55 ++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 8bddafcf..0ca9b539 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -13,7 +13,8 @@ KBASE_PARAM_PATH = "links.add_path" -KBASE_PARAM_DOMAIN_REGEX = "links.regex" +KBASE_PARAM_DOMAIN_REGEX = "links.domain" +KBASE_PARAM_REGEX = "links.regex" KBASE_NEW_DOMAIN = "links.new_domains" @@ -28,8 +29,24 @@ class links(BasePlugin, DiscoveryPluginMixin): priority = 99 parameters = ( - ("add_path", False, False, "Add parsed paths as results."), - ("regex", None, False, "Regex of accepted domains."), + ( + "add_path", + False, + False, + "Re-enqueue found paths. ie. /path/link.html link includes also path/", + ), + ( + "domain", + None, + False, + "Regex of accepted domains tested against url.netloc. This is useful for restricting crawling certain domains.", + ), + ( + "regex", + None, + False, + "Regex of accepted links tested against the full url. If domain is not set and regex is, domain defaults to .*. This is useful for restricting crawling certain file types.", + ), ) def __init__(self): @@ -58,8 +75,18 @@ def __init__(self): self.domain_regex = None if self.kbase[KBASE_PARAM_DOMAIN_REGEX][0]: self.domain_regex = re.compile( - self.kbase[KBASE_PARAM_DOMAIN_REGEX][0], re.MULTILINE | re.DOTALL + self.kbase[KBASE_PARAM_DOMAIN_REGEX][0], re.IGNORECASE ) + + self.regex_param = None + if self.kbase[KBASE_PARAM_REGEX][0]: + self.regex_param = re.compile( + self.kbase[KBASE_PARAM_REGEX][0], re.IGNORECASE + ) + + if self.regex_param and self.domain_regex is None: + self.domain_regex = re.compile(".*", re.IGNORECASE) + self.list_links = set() def validate(self, fuzzresult): @@ -104,17 +131,19 @@ def enqueue_link(self, fuzzresult, link_url, parsed_link): self.queue_url(urljoin(fuzzresult.url, newpath)) # file path - self.queue_url(urljoin(fuzzresult.url, link_url)) + new_link = urljoin(fuzzresult.url, link_url) + + if not self.regex_param or ( + self.regex_param and self.regex_param.search(new_link) is not None + ): + self.queue_url(new_link) + self.add_verbose_result("link", "New link found", new_link) def from_domain(self, fuzzresult, parsed_link): # relative path if not parsed_link.netloc and parsed_link.path: return True - # same domain - if parsed_link.netloc == self.base_fuzz_res.history.urlp.netloc: - return True - # regex domain if ( self.domain_regex @@ -122,11 +151,15 @@ def from_domain(self, fuzzresult, parsed_link): ): return True + # same domain + if parsed_link.netloc == self.base_fuzz_res.history.urlp.netloc: + return True + if ( parsed_link.netloc and parsed_link.netloc not in self.kbase[KBASE_NEW_DOMAIN] ): self.kbase[KBASE_NEW_DOMAIN].append(parsed_link.netloc) - self.add_result( - "domain", "New domain found, link not enqueued", parsed_link.netloc + self.add_verbose_result( + "domain", "New domain found (link not enqueued)", parsed_link.netloc ) From 53f64957817b0f6635992750e50d31c95f60b6a4 Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 20:36:56 +0200 Subject: [PATCH 29/89] headers in one plugin --- src/wfuzz/plugins/scripts/headers.py | 179 +++++++++++++++--- src/wfuzz/plugins/scripts/uncommon_headers.py | 96 ---------- 2 files changed, 156 insertions(+), 119 deletions(-) delete mode 100644 src/wfuzz/plugins/scripts/uncommon_headers.py diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index e12dfbd4..94b30e30 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -1,8 +1,99 @@ from wfuzz.plugin_api.base import BasePlugin from wfuzz.externals.moduleman.plugin import moduleman_plugin +import re -KBASE_KEY = "http.response.headers.servers" +KBASE_KEY = "http.servers" +KBASE_KEY_RESP_UNCOMMON = "http.response.headers.uncommon" +KBASE_KEY_REQ_UNCOMMON = "http.request.headers.uncommon" + +SERVER_HEADERS = ["server", "x-powered-by" "via"] + +COMMON_RESPONSE_HEADERS_REGEX_LIST = [ + r"^Server$", + r"^X-Powered-By$", + r"^Via$", + r"^Access-Control.*$", + r"^Accept-.*$", + r"^age$", + r"^allow$", + r"^Cache-control$", + r"^Client-.*$", + r"^Connection$", + r"^Content-.*$", + r"^Cross-Origin-Resource-Policy$", + r"^Date$", + r"^Etag$", + r"^Expires$", + r"^Keep-Alive$", + r"^Last-Modified$", + r"^Link$", + r"^Location$", + r"^P3P$", + r"^Pragma$", + r"^Proxy-.*$", + r"^Refresh$", + r"^Retry-After$", + r"^Referrer-Policy$", + r"^Set-Cookie$", + r"^Server-Timing$", + r"^Status$", + r"^Strict-Transport-Security$", + r"^Timing-Allow-Origin$", + r"^Trailer$", + r"^Transfer-Encoding$", + r"^Upgrade$", + r"^Vary$", + r"^Warning^$", + r"^WWW-Authenticate$", + r"^X-Content-Type-Options$", + r"^X-Download-Options$", + r"^X-Frame-Options$", + r"^X-Microsite$", + r"^X-Request-Handler-Origin-Region$", + r"^X-XSS-Protection$", +] + +COMMON_RESPONSE_HEADERS_REGEX = re.compile( + "({})".format("|".join(COMMON_RESPONSE_HEADERS_REGEX_LIST)), re.IGNORECASE +) + +COMMON_REQ_HEADERS_REGEX_LIST = [ + r"A-IM$", + r"Accept$", + r"Accept-.*$", + r"Access-Control-.*$", + r"Authorization$", + r"Cache-Control$", + r"Connection$", + r"Content-.*$", + r"Cookie$", + r"Date$", + r"Expect$", + r"Forwarded$", + r"From$", + r"Host$", + r"If-.*$", + r"Max-Forwards$", + r"Origin$", + r"Pragma$", + r"Proxy-Authorization$", + r"Range$", + r"Referer$", + r"TE$", + r"User-Agent$", + r"Upgrade$", + r"Upgrade-Insecure-Requests$", + r"Via$", + r"Warning$", + r"X-Requested-With$", + r"X-HTTP-Method-Override$", + r"X-Requested-With$", +] + +COMMON_REQ_HEADERS_REGEX = re.compile( + "({})".format("|".join(COMMON_REQ_HEADERS_REGEX_LIST)), re.IGNORECASE +) @moduleman_plugin @@ -10,10 +101,14 @@ class headers(BasePlugin): name = "headers" author = ("Xavi Mendez (@xmendez)",) version = "0.1" - summary = ( - "Looks for new response headers associated to HTTP servers in HTTP responses" + summary = "Looks for HTTP headers." + description = ( + "Looks for NEW HTTP headers:", + "\t- Response HTTP headers associated to web servers.", + "\t- Uncommon response HTTP headers.", + "\t- Uncommon request HTTP headers.", + "It is worth noting that, only the FIRST match of the above headers is registered.", ) - description = ("Looks for new server headers",) category = ["info", "passive", "default"] priority = 99 parameters = () @@ -21,27 +116,65 @@ class headers(BasePlugin): def __init__(self): BasePlugin.__init__(self) - self.watch_headers = ["Server", "X-Powered-By" "Via"] - def validate(self, fuzzresult): return True + def check_request_header(self, fuzzresult, header, value): + header_value = None + if not COMMON_REQ_HEADERS_REGEX.match(header): + header_value = header + + if header_value is not None: + if ( + header_value.lower() not in self.kbase[KBASE_KEY_REQ_UNCOMMON] + or KBASE_KEY_REQ_UNCOMMON not in self.kbase + ): + self.add_verbose_result( + "reqheader", + "New uncommon HTTP request header", + "{}: {}".format(header_value, value), + ) + + self.kbase[KBASE_KEY_REQ_UNCOMMON].append(header_value.lower()) + + def check_response_header(self, fuzzresult, header, value): + header_value = None + if not COMMON_RESPONSE_HEADERS_REGEX.match(header): + header_value = header + + if header_value is not None: + if ( + header_value.lower() not in self.kbase[KBASE_KEY_RESP_UNCOMMON] + or KBASE_KEY_RESP_UNCOMMON not in self.kbase + ): + self.add_verbose_result( + "header", + "New uncommon HTTP response header", + "{}: {}".format( + header_value, fuzzresult.history.headers.response[header_value], + ), + ) + + self.kbase[KBASE_KEY_RESP_UNCOMMON].append(header_value.lower()) + + def check_server_header(self, fuzzresult, header, value): + if header.lower() in SERVER_HEADERS: + if ( + value.lower() not in self.kbase[KBASE_KEY] + or KBASE_KEY not in self.kbase + ): + self.add_verbose_result( + "server", + "New server HTTP response header", + "{}: {}".format(header, value), + ) + + self.kbase[KBASE_KEY].append(value.lower()) + def process(self, fuzzresult): + for header, value in fuzzresult.history.headers.request.items(): + self.check_request_header(fuzzresult, header, value) - for header in self.watch_headers: - header_value = None - if header in fuzzresult.history.headers.response: - header_value = fuzzresult.history.headers.response[header] - - if header_value is not None: - if ( - header_value.lower() not in self.kbase[KBASE_KEY] - or KBASE_KEY not in self.kbase - ): - self.add_result( - "header", - "New server response header", - "{}: {}".format(header, header_value), - ) - - self.kbase[KBASE_KEY].append(header_value.lower()) + for header, value in fuzzresult.history.headers.response.items(): + self.check_response_header(fuzzresult, header, value) + self.check_server_header(fuzzresult, header, value) diff --git a/src/wfuzz/plugins/scripts/uncommon_headers.py b/src/wfuzz/plugins/scripts/uncommon_headers.py deleted file mode 100644 index 0778134f..00000000 --- a/src/wfuzz/plugins/scripts/uncommon_headers.py +++ /dev/null @@ -1,96 +0,0 @@ -from wfuzz.plugin_api.base import BasePlugin -from wfuzz.externals.moduleman.plugin import moduleman_plugin - -import re - - -KBASE_KEY_UNCOMMON = "http.response.headers.uncommon" - - -@moduleman_plugin -class unheaders(BasePlugin): - name = "unheaders" - author = ("Xavi Mendez (@xmendez)",) - version = "0.1" - summary = "Looks for new uncommon reponse headers" - description = ("Looks for new uncommon http reponse headers",) - category = ["info", "passive", "verbose"] - priority = 99 - parameters = () - - def __init__(self): - BasePlugin.__init__(self) - - common_headers_regex = [ - r"^Server$", - r"^X-Powered-By$", - r"^Via$", - r"^Access-Control.*$", - r"^Accept-.*$", - r"^age$", - r"^allow$", - r"^Cache-control$", - r"^Client-.*$", - r"^Connection$", - r"^Content-.*$", - r"^Cross-Origin-Resource-Policy$", - r"^Date$", - r"^Etag$", - r"^Expires$", - r"^Keep-Alive$", - r"^Last-Modified$", - r"^Link$", - r"^Location$", - r"^P3P$", - r"^Pragma$", - r"^Proxy-.*$", - r"^Refresh$", - r"^Retry-After$", - r"^Referrer-Policy$", - r"^Set-Cookie$", - r"^Server-Timing$", - r"^Status$", - r"^Strict-Transport-Security$", - r"^Timing-Allow-Origin$", - r"^Trailer$", - r"^Transfer-Encoding$", - r"^Upgrade$", - r"^Vary$", - r"^Warning^$", - r"^WWW-Authenticate$", - r"^X-Content-Type-Options$", - r"^X-Download-Options$", - r"^X-Frame-Options$", - r"^X-Microsite$", - r"^X-Request-Handler-Origin-Region$", - r"^X-XSS-Protection$", - ] - - self.common_headers_regex = re.compile( - "({})".format("|".join(common_headers_regex)), re.IGNORECASE - ) - - def validate(self, fuzzresult): - return True - - def process(self, fuzzresult): - for header in fuzzresult.history.headers.response.keys(): - header_value = None - if not self.common_headers_regex.match(header): - header_value = header - - if header_value is not None: - if ( - header_value.lower() not in self.kbase[KBASE_KEY_UNCOMMON] - or KBASE_KEY_UNCOMMON not in self.kbase - ): - self.add_result( - "header", - "New uncommon reponse header", - "{}: {}".format( - header_value, - fuzzresult.history.headers.response[header_value], - ), - ) - - self.kbase[KBASE_KEY_UNCOMMON].append(header_value.lower()) From 974249242b5260a7c0a08a121b296c65dabfaf1a Mon Sep 17 00:00:00 2001 From: javi Date: Sat, 10 Oct 2020 20:38:24 +0200 Subject: [PATCH 30/89] not show summary when verbose --- src/wfuzz/fuzzobjects.py | 1 + src/wfuzz/fuzzqueues.py | 4 ++-- src/wfuzz/ui/console/mvc.py | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index 1c14b0e3..d539cfc2 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -411,6 +411,7 @@ def update_from_options(self, options): class FuzzPlugin(FuzzItem): OUTPUT_CAT = "output" + SUMMARY_TYPE = "summary" def __init__(self): FuzzItem.__init__(self, FuzzType.PLUGIN) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index b9c881a8..21f69b29 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -367,7 +367,7 @@ def process_results(self, res, plugins_res_queue): plugin_factory.create( "plugin_from_finding", FuzzPlugin.OUTPUT_CAT, - "msg", + FuzzPlugin.SUMMARY_TYPE, "Plugin {}: {} new {}(s) found.".format( plugin_name, enq_num, domain ), @@ -399,7 +399,7 @@ def process(self, fuzz_res): plugin_factory.create( "plugin_from_finding", FuzzPlugin.OUTPUT_CAT, - "msg", + FuzzPlugin.SUMMARY_TYPE, "Enqueued response for recursion (level=%d)" % (seed.rlevel), None, False, diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 28340fbd..a595e09f 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -8,7 +8,7 @@ except ImportError: from itertools import izip_longest as zip_longest -from wfuzz.fuzzobjects import FuzzWordType, FuzzType +from wfuzz.fuzzobjects import FuzzWordType, FuzzType, FuzzPlugin from .common import exec_banner, Term from .getch import _Getch @@ -325,7 +325,9 @@ def result(self, res): if res.plugins_res: for plugin_res in res.plugins_res: - if plugin_res._verbose and not self.verbose: + if (plugin_res._verbose and not self.verbose) or ( + plugin_res.itype == FuzzPlugin.SUMMARY_TYPE and self.verbose + ): continue sys.stdout.write( From 017b125608933bd411958e55fc5076097d28414b Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 11 Oct 2020 00:18:57 +0200 Subject: [PATCH 31/89] recursive when transport and rlevel --- src/wfuzz/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/core.py b/src/wfuzz/core.py index 8f07e8ba..01180c8a 100644 --- a/src/wfuzz/core.py +++ b/src/wfuzz/core.py @@ -56,8 +56,10 @@ def __init__(self, options): if options.get("script"): self.qmanager.add("plugins_queue", JobQ(options)) - if options.get("script") or options.get("rlevel") > 0: + if options.get("rlevel") > 0: self.qmanager.add("recursive_queue", RecursiveQ(options)) + + if (options.get("script") or options.get("rlevel") > 0) and options.get("transport") == "http": rq = RoutingQ( options, { From 0e74b475643b92455862e220e67ad5d92e12d901 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 13 Oct 2020 13:53:21 +0200 Subject: [PATCH 32/89] only add routingq when transport is http --- src/wfuzz/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/core.py b/src/wfuzz/core.py index 01180c8a..8b630e05 100644 --- a/src/wfuzz/core.py +++ b/src/wfuzz/core.py @@ -59,7 +59,9 @@ def __init__(self, options): if options.get("rlevel") > 0: self.qmanager.add("recursive_queue", RecursiveQ(options)) - if (options.get("script") or options.get("rlevel") > 0) and options.get("transport") == "http": + if (options.get("script") or options.get("rlevel") > 0) and options.get( + "transport" + ) == "http": rq = RoutingQ( options, { From fc738a1d88b28828cd74d12f4517904b3a1b811c Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 13 Oct 2020 20:20:07 +0200 Subject: [PATCH 33/89] remove discarded type --- src/wfuzz/factories/fuzzresfactory.py | 4 ++++ src/wfuzz/fuzzobjects.py | 15 +++--------- src/wfuzz/fuzzqueues.py | 23 ++++++++----------- src/wfuzz/myqueues.py | 33 ++++++++++++--------------- src/wfuzz/ui/console/mvc.py | 2 +- 5 files changed, 32 insertions(+), 45 deletions(-) diff --git a/src/wfuzz/factories/fuzzresfactory.py b/src/wfuzz/factories/fuzzresfactory.py index 3ddd5828..4734c32a 100644 --- a/src/wfuzz/factories/fuzzresfactory.py +++ b/src/wfuzz/factories/fuzzresfactory.py @@ -27,6 +27,7 @@ class FuzzResultDictioBuilder: def __call__(self, options, dictio_item): res = copy.deepcopy(options["compiled_seed"]) res.item_type = FuzzType.RESULT + res.discarded = False res.payload_man.update_from_dictio(dictio_item) res.update_from_options(options) @@ -69,6 +70,7 @@ class FuzzResultAllVarBuilder: def __call__(self, options, var_name, payload): fuzzres = copy.deepcopy(options["compiled_seed"]) fuzzres.item_type = FuzzType.RESULT + fuzzres.discarded = False fuzzres.payload_man = payman_factory.create("empty_payloadman", payload) fuzzres.payload_man.update_from_dictio([payload]) fuzzres.history.wf_allvars_set = {var_name: payload.content} @@ -97,6 +99,7 @@ def __call__(self, seed): new_seed.rlevel_desc += " - " new_seed.rlevel_desc += seed.payload_man.description() new_seed.item_type = FuzzType.SEED + new_seed.discarded = False new_seed.payload_man = payman_factory.create( "payloadman_from_request", new_seed.history ) @@ -113,6 +116,7 @@ def __call__(self, seed, url): fr.rlevel_desc += " - " fr.rlevel_desc += seed.payload_man.description() fr.item_type = FuzzType.BACKFEED + fr.discarded = False fr.is_baseline = False fr.payload_man = payman_factory.create( diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index d539cfc2..fa91f337 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -24,17 +24,7 @@ class FuzzWordType(Enum): class FuzzType(Enum): - ( - SEED, - BACKFEED, - RESULT, - ERROR, - STARTSEED, - ENDSEED, - CANCEL, - DISCARDED, - PLUGIN, - ) = range(9) + (SEED, BACKFEED, RESULT, ERROR, STARTSEED, ENDSEED, CANCEL, PLUGIN,) = range(8) class FuzzItem(object): @@ -43,6 +33,8 @@ class FuzzItem(object): def __init__(self, item_type): self.item_id = next(FuzzItem.newid) self.item_type = item_type + self.rlevel = 1 + self.discarded = False def __str__(self): return "FuzzItem, type: {}".format(self.item_type.name) @@ -279,7 +271,6 @@ def __init__(self, history=None, exception=None, track_id=True): self.exception = exception self.is_baseline = False - self.rlevel = 1 self.rlevel_desc = "" self.nres = next(FuzzResult.newid) if track_id else 0 diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index 21f69b29..227a68e2 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -33,8 +33,8 @@ def get_name(self): def cancel(self): self.options["compiled_stats"].cancelled = True - def items_to_process(self, item): - return item.item_type in [FuzzType.STARTSEED] + def items_to_process(self): + return [FuzzType.STARTSEED] def process(self, item): self.stats.pending_seeds.inc() @@ -65,8 +65,8 @@ def get_name(self): def cancel(self): self.options["compiled_stats"].cancelled = True - def items_to_process(self, item): - return item.item_type in [FuzzType.STARTSEED, FuzzType.SEED] + def items_to_process(self): + return [FuzzType.STARTSEED, FuzzType.SEED] def send_baseline(self): fuzz_baseline = self.options["compiled_baseline"] @@ -160,9 +160,6 @@ def __init__(self, options): def mystart(self): self.printer.header(self.stats) - def items_to_process(self, item): - return item.item_type in [FuzzType.RESULT] - def get_name(self): return "ConsolePrinterQ" @@ -182,8 +179,8 @@ def __init__(self, options): def mystart(self): self.printer.header(self.stats) - def items_to_process(self, item): - return item.item_type in [FuzzType.RESULT, FuzzType.DISCARDED] + def process_discarded(self): + return True def get_name(self): return "CLIPrinterQ" @@ -222,8 +219,8 @@ def __init__(self, options, routes): def get_name(self): return "RoutingQ" - def items_to_process(self, item): - return item.item_type in [FuzzType.SEED, FuzzType.BACKFEED] + def items_to_process(self): + return [FuzzType.SEED, FuzzType.BACKFEED] def process(self, item): if item.item_type in self.routes: @@ -467,8 +464,8 @@ def _cleanup(self): self.http_pool.deregister() self.exit_job = True - def items_to_process(self, item): - return item.item_type in [FuzzType.RESULT, FuzzType.BACKFEED] + def items_to_process(self): + return [FuzzType.RESULT, FuzzType.BACKFEED] def process(self, obj): self.pause.wait() diff --git a/src/wfuzz/myqueues.py b/src/wfuzz/myqueues.py index 620b71c2..7c7d7399 100644 --- a/src/wfuzz/myqueues.py +++ b/src/wfuzz/myqueues.py @@ -60,8 +60,11 @@ def process(self, item): def get_name(self): raise NotImplementedError - def items_to_process(self, item): - return item.item_type in [FuzzType.RESULT] + def process_discarded(self): + return False + + def items_to_process(self): + return [FuzzType.RESULT] # Override this method if needed. This will be called just before cancelling the job. def cancel(self): @@ -91,13 +94,8 @@ def send(self, item): self.queue_out.put(item) def discard(self, item): - if item.item_type == FuzzType.RESULT: - item.item_type = FuzzType.DISCARDED - self.send(item) - else: - raise FuzzExceptInternalError( - FuzzException.FATAL, "Only results can be discarded" - ) + item.discarded = True + self.send(item) def join(self): MyPriorityQueue.join(self) @@ -143,7 +141,9 @@ def run(self): self.task_done() continue - if self.items_to_process(item): + if ( + not item.discarded or (item.discarded and self.process_discarded()) + ) and item.item_type in self.items_to_process(): self.process(item) else: self.send(item) @@ -160,8 +160,6 @@ class LastFuzzQueue(FuzzQueue): def __init__(self, options, queue_out=None, limit=0): FuzzQueue.__init__(self, options, queue_out, limit) - self.items_to_send = [FuzzType.RESULT] - def get_name(self): return "LastFuzzQueue" @@ -171,10 +169,6 @@ def process(self): def _cleanup(self): pass - def send(self, item): - if item.item_type in self.items_to_send: - self.queue_out.put(item) - def _throw(self, e): self.queue_out.put_first(FuzzError(e)) @@ -199,14 +193,15 @@ def run(self): cancelling = True continue - self.send(item) + if item.item_type == FuzzType.RESULT and not item.discarded: + self.send(item) if item.item_type == FuzzType.ENDSEED: self.stats.pending_seeds.dec() - elif item.item_type in [FuzzType.RESULT, FuzzType.DISCARDED]: + elif item.item_type == FuzzType.RESULT: self.stats.processed.inc() self.stats.pending_fuzz.dec() - if item.item_type == FuzzType.DISCARDED: + if item.discarded: self.stats.filtered.inc() if self.stats.pending_fuzz() == 0 and self.stats.pending_seeds() == 0: diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index a595e09f..6e7806f2 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -303,7 +303,7 @@ def result(self, res): else: self._print(res) - if res.item_type == FuzzType.RESULT: + if not res.discarded: if ( self.previous and res.payload_man From a523de193509de62bbb27a4d6b4c14a9037ee2cb Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 14 Oct 2020 19:12:25 +0200 Subject: [PATCH 34/89] fix interact q --- src/wfuzz/core.py | 1 + src/wfuzz/ui/console/mvc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/core.py b/src/wfuzz/core.py index 8b630e05..00e89eaf 100644 --- a/src/wfuzz/core.py +++ b/src/wfuzz/core.py @@ -30,6 +30,7 @@ def __init__(self, options): # genReq ---> seed_queue -> [slice_queue] -> http_queue/dryrun -> [round_robin -> plugins_queue] * N # -> [recursive_queue -> routing_queue] -> [filter_queue] -> [save_queue] -> [printer_queue] ---> results + self.options = options self.qmanager = QueueManager(options) self.results_queue = MyPriorityQueue() diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 6e7806f2..6ad2ca8a 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -92,8 +92,8 @@ def __init__(self, fuzzer, view): # dynamic keyboard bindings def on_exit(self, **event): self.fuzzer.cancel_job() - self.fuzzer.genReq.stats.mark_end() self.view.cancel_job() + self.fuzzer.options.close() def on_help(self, **event): print(usage) From 1106b9e1182607c8d3099eecf42a4c2dd9717b56 Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 14 Oct 2020 19:21:02 +0200 Subject: [PATCH 35/89] name all seed queues, seed_queue --- src/wfuzz/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/core.py b/src/wfuzz/core.py index 00e89eaf..2203dc45 100644 --- a/src/wfuzz/core.py +++ b/src/wfuzz/core.py @@ -35,7 +35,7 @@ def __init__(self, options): self.results_queue = MyPriorityQueue() if options["allvars"]: - self.qmanager.add("allvars_queue", AllVarQ(options)) + self.qmanager.add("seed_queue", AllVarQ(options)) else: self.qmanager.add("seed_queue", SeedQ(options)) From 24be728ee73bf595ea2ba0d092c07fbd6f5608af Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 14 Oct 2020 20:04:50 +0200 Subject: [PATCH 36/89] fix interactive s --- src/wfuzz/core.py | 2 +- src/wfuzz/fuzzobjects.py | 8 ++-- src/wfuzz/myhttp.py | 2 +- src/wfuzz/ui/console/mvc.py | 81 ++++++++++++++++++------------------- 4 files changed, 45 insertions(+), 48 deletions(-) diff --git a/src/wfuzz/core.py b/src/wfuzz/core.py index 2203dc45..947251df 100644 --- a/src/wfuzz/core.py +++ b/src/wfuzz/core.py @@ -120,7 +120,7 @@ def __next__(self): def stats(self): return dict( list(self.qmanager.get_stats().items()) - + list(self.qmanager["transport_queue"].job_stats().items()) + + list(self.qmanager["transport_queue"].http_pool.job_stats().items()) + list(self.options.stats.get_stats().items()) ) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index fa91f337..cc5c87c9 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -92,11 +92,11 @@ def get_stats(self): "url": self.url, "total": self.total_req, "backfed": self.backfeed(), - "Processed": self.processed(), - "Pending": self.pending_fuzz(), + "processed": self.processed(), + "pending": self.pending_fuzz(), "filtered": self.filtered(), - "Pending_seeds": self.pending_seeds(), - "totaltime": self._totaltime, + "pending_seeds": self.pending_seeds(), + "totaltime": self.totaltime, } def mark_start(self): diff --git a/src/wfuzz/myhttp.py b/src/wfuzz/myhttp.py index 948cd76f..1cb58695 100644 --- a/src/wfuzz/myhttp.py +++ b/src/wfuzz/myhttp.py @@ -68,7 +68,7 @@ def job_stats(self): with self.mutex_stats: dic = { "http_processed": self.processed, - "http_registered": len(self._registered), + "http_registered": self._registered, } return dic diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 6ad2ca8a..983669a5 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -83,6 +83,7 @@ def __init__(self, fuzzer, view): self.fuzzer = fuzzer self.view = view self.__paused = False + self.stats = fuzzer.options.get("compiled_stats") self.view.dispatcher.subscribe(self.on_help, "?") self.view.dispatcher.subscribe(self.on_pause, "p") @@ -102,56 +103,52 @@ def on_pause(self, **event): self.__paused = not self.__paused if self.__paused: self.fuzzer.pause_job() - - if self._debug: - print("\n=============== Paused ==================") - stats = self.fuzzer.stats() - for k, v in list(stats.items()): - print("%s: %s" % (k, v)) - print("\n=========================================") else: self.fuzzer.resume_job() def on_stats(self, **event): if self._debug: - print("\n=============== Paused ==================") - stats = self.fuzzer.stats() - for k, v in list(stats.items()): - print("%s: %s" % (k, v)) - print("\n=========================================") + self.show_debug_stats() else: - pending = ( - self.fuzzer.genReq.stats.total_req - - self.fuzzer.genReq.stats.processed() - ) - summary = self.fuzzer.genReq.stats - summary.mark_end() - print("\nTotal requests: %s\r" % str(summary.total_req)) - print("Pending requests: %s\r" % str(pending)) - - if summary.backfeed() > 0: - print( - "Processed Requests: %s (%d + %d)\r" - % ( - str(summary.processed())[:8], - (summary.processed() - summary.backfeed()), - summary.backfeed(), - ) + self.show_stats() + + def show_debug_stats(self): + print("\n=============== Paused ==================") + stats = self.stats.get_stats() + for k, v in list(stats.items()): + print("%s: %s" % (k, v)) + print("\n=========================================") + + def show_stats(self): + pending = self.stats.total_req - self.stats.processed() + summary = self.stats + summary.mark_end() + print("\nTotal requests: %s\r" % str(summary.total_req)) + print("Pending requests: %s\r" % str(pending)) + + if summary.backfeed() > 0: + print( + "Processed Requests: %s (%d + %d)\r" + % ( + str(summary.processed())[:8], + (summary.processed() - summary.backfeed()), + summary.backfeed(), ) - else: - print("Processed Requests: %s\r" % (str(summary.processed())[:8])) - print("Filtered Requests: %s\r" % (str(summary.filtered())[:8])) - req_sec = ( - summary.processed() / summary.totaltime if summary.totaltime > 0 else 0 ) - print("Total time: %s\r" % str(summary.totaltime)[:8]) - if req_sec > 0: - print("Requests/sec.: %s\r" % str(req_sec)[:8]) - eta = pending / req_sec - if eta > 60: - print("ET left min.: %s\r\n" % str(eta / 60)[:8]) - else: - print("ET left sec.: %s\r\n" % str(eta)[:8]) + else: + print("Processed Requests: %s\r" % (str(summary.processed())[:8])) + print("Filtered Requests: %s\r" % (str(summary.filtered())[:8])) + req_sec = ( + summary.processed() / summary.totaltime if summary.totaltime > 0 else 0 + ) + print("Total time: %s\r" % str(summary.totaltime)[:8]) + if req_sec > 0: + print("Requests/sec.: %s\r" % str(req_sec)[:8]) + eta = pending / req_sec + if eta > 60: + print("ET left min.: %s\r\n" % str(eta / 60)[:8]) + else: + print("ET left sec.: %s\r\n" % str(eta)[:8]) class View: From 1b34d05c60c5e199772675048014f68edacad699 Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 14 Oct 2020 20:09:27 +0200 Subject: [PATCH 37/89] partial time in get stats --- src/wfuzz/fuzzobjects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index cc5c87c9..d3aff3fa 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -96,7 +96,7 @@ def get_stats(self): "pending": self.pending_fuzz(), "filtered": self.filtered(), "pending_seeds": self.pending_seeds(), - "totaltime": self.totaltime, + "totaltime": time.time() - self.__starttime, } def mark_start(self): From 95efe6825dc14dca8568d77d52dc6f2d201212c7 Mon Sep 17 00:00:00 2001 From: javi Date: Wed, 14 Oct 2020 20:31:29 +0200 Subject: [PATCH 38/89] get stats from fuzzer --- src/wfuzz/ui/console/mvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 983669a5..51b79f91 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -114,7 +114,7 @@ def on_stats(self, **event): def show_debug_stats(self): print("\n=============== Paused ==================") - stats = self.stats.get_stats() + stats = self.fuzzer.stats() for k, v in list(stats.items()): print("%s: %s" % (k, v)) print("\n=========================================") From f563ad641e235c1b0b18835d351aeda19dccc99a Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 19 Oct 2020 21:00:41 +0200 Subject: [PATCH 39/89] remove httpreceiver queue limit --- src/wfuzz/fuzzqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index 227a68e2..81f2ce64 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -482,7 +482,7 @@ def __read_http_results(self): class HttpReceiver(FuzzQueue): def __init__(self, options): - FuzzQueue.__init__(self, options, limit=options.get("concurrent") * 5) + FuzzQueue.__init__(self, options) def get_name(self): return "HttpReceiver" From 5d7f8748cd8ff54e7d89249cb3824ba954ad709e Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 19 Oct 2020 21:23:03 +0200 Subject: [PATCH 40/89] basic queue size profiling --- src/wfuzz/wfuzz.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index fe84dc6a..68d52996 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -15,6 +15,25 @@ from .fuzzobjects import FuzzWordType +PROFILING = False + + +def print_profiling(profiling_list, profiling_header): + avg = [float(sum(col)) / len(col) for col in list(zip(*profiling_list))] + maxx = [max(col) for col in list(zip(*profiling_list))] + + print( + ", ".join( + ["{}={}".format(pair[0], pair[1]) for pair in zip(profiling_header, avg)] + ) + ) + print( + ", ".join( + ["{}={}".format(pair[0], pair[1]) for pair in zip(profiling_header, maxx)] + ) + ) + + def main(): kb = None fz = None @@ -41,8 +60,19 @@ def main(): Controller(fz, kb) kb.start() + if PROFILING: + profiling_header = list(fz.qmanager._queues.keys()) + profiling_list = [] + for res in fz: - pass + if PROFILING: + profiling = list(fz.qmanager.get_stats().items()) + profiling_list.append([pair[1] for pair in profiling]) + else: + pass + + if PROFILING: + print_profiling(profiling_list, profiling_header) except FuzzException as e: warnings.warn("Fatal exception: {}".format(str(e))) except KeyboardInterrupt: From b517e205c34df8bffb688efff0c17539d0d7e536 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 19 Oct 2020 22:16:48 +0200 Subject: [PATCH 41/89] add lower in comparison --- src/wfuzz/filters/ppfilter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/filters/ppfilter.py b/src/wfuzz/filters/ppfilter.py index 2eef5bfe..13032540 100644 --- a/src/wfuzz/filters/ppfilter.py +++ b/src/wfuzz/filters/ppfilter.py @@ -280,7 +280,7 @@ def __compute_expr(self, tokens): elif isinstance(leftvalue, list): ret = value_in_any_list_item(rightvalue, leftvalue) elif isinstance(leftvalue, dict) or isinstance(leftvalue, DotDict): - ret = rightvalue.lower() in str(leftvalue) + ret = rightvalue.lower() in str(leftvalue).lower() else: raise FuzzExceptBadOptions( "Invalid operand type {}".format(rightvalue) From ad3b9c06e2899071e221ba47a02ff179683a6af1 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 19 Oct 2020 22:35:01 +0200 Subject: [PATCH 42/89] dont print empty values --- src/wfuzz/wfuzz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index 68d52996..eee2aac2 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -158,7 +158,9 @@ def usage(): if payload_type == FuzzWordType.WORD: print(res.description) elif payload_type == FuzzWordType.FUZZRES and session_options["show_field"]: - print(res._field("\n")) + field_to_print = res._field("\n") + if field_to_print: + print(field_to_print) except KeyboardInterrupt: pass From 234d6591046d6f7bbdc9538a80450d7df2ee915a Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 19 Oct 2020 23:06:31 +0200 Subject: [PATCH 43/89] fix prev in wfpayload --- src/wfuzz/fuzzqueues.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index 81f2ce64..acfe2786 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -419,9 +419,10 @@ def process(self, item): if item.payload_man.get_payload_type(1) == FuzzWordType.FUZZRES: item = item.payload_man.get_payload_content(1) item.update_from_options(self.options) - item.payload_man = payman_factory.create( - "empty_payloadman", FuzzWord(item.url, FuzzWordType.WORD) - ) + if not item.payload_man: + item.payload_man = payman_factory.create( + "empty_payloadman", FuzzWord(item.url, FuzzWordType.WORD) + ) self.send(item) From 56e1c16333d25ee457d815fba29f75bce50751f4 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 20 Oct 2020 00:38:39 +0200 Subject: [PATCH 44/89] change npm_deps to verbose result --- src/wfuzz/plugins/scripts/npm_deps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wfuzz/plugins/scripts/npm_deps.py b/src/wfuzz/plugins/scripts/npm_deps.py index dde3156a..10fda31a 100644 --- a/src/wfuzz/plugins/scripts/npm_deps.py +++ b/src/wfuzz/plugins/scripts/npm_deps.py @@ -42,8 +42,8 @@ def validate(self, fuzzresult): def process(self, fuzzresult): if self.match_dev: for name, version in self.REGEX_PATT.findall(self.match_dev.group(1)): - self.add_result("dependency", "npm dependency", name) + self.add_verbose_result("dependency", "npm dependency", name) if self.match: for name, version in self.REGEX_PATT.findall(self.match.group(1)): - self.add_result("dev_dependency", "npm dev dependency", name) + self.add_verbose_result("dev_dependency", "npm dev dependency", name) From 679124cfdab82f0250d7b4e1f80afa3d16ac89c5 Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 00:50:36 +0200 Subject: [PATCH 45/89] refactor plugin-help --- src/wfuzz/ui/console/clparser.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/wfuzz/ui/console/clparser.py b/src/wfuzz/ui/console/clparser.py index 7d61e74c..d5980c4d 100644 --- a/src/wfuzz/ui/console/clparser.py +++ b/src/wfuzz/ui/console/clparser.py @@ -129,16 +129,13 @@ def show_plugin_ext_help(self, registrant, category="$all$"): for desc_lines in plugin.description: print(" %s" % desc_lines) print("Parameters:") - for param in plugin.parameters: - print( - " %s %s%s: %s" - % ( - "+" if param[2] else "-", - param[0], - " (= %s)" % str(param[1]) if param[1] else "", - param[3], - ) - ) + for name, default_value, mandatory, description in plugin.parameters: + print(" {} {}{}: {}".format( + "+" if mandatory else "-", + name, + " (= %s)" % str(default_value) if default_value is not None else "", + description, + )) print("\n") sys.exit(0) From 7dbc014137fbfe5b10953d63049253e991bba27b Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 00:51:59 +0200 Subject: [PATCH 46/89] enqueue param in links plugin --- src/wfuzz/plugin_api/base.py | 4 ++++ src/wfuzz/plugins/scripts/links.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index feca4fea..4ac05e2f 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -10,6 +10,7 @@ import sys import os +from distutils import util # python 2 and 3: iterator from builtins import object @@ -79,6 +80,9 @@ def queue_url(self, url): ) ) + def _bool(self, value): + return bool(util.strtobool(value)) + class BasePrinter: def __init__(self, output): diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 0ca9b539..614fd634 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -13,6 +13,7 @@ KBASE_PARAM_PATH = "links.add_path" +KBASE_PARAM_ENQUEUE = "links.enqueue" KBASE_PARAM_DOMAIN_REGEX = "links.domain" KBASE_PARAM_REGEX = "links.regex" KBASE_NEW_DOMAIN = "links.new_domains" @@ -30,10 +31,16 @@ class links(BasePlugin, DiscoveryPluginMixin): parameters = ( ( - "add_path", + "enqueue", + "True", False, + "If True, enqueue found links.", + ), + ( + "add_path", + "False", False, - "Re-enqueue found paths. ie. /path/link.html link includes also path/", + "if True, re-enqueue found paths. ie. /path/link.html link enqueues also /path/", ), ( "domain", @@ -70,7 +77,8 @@ def __init__(self): ("Location", re.compile(r"(.*)")), ] - self.add_path = self.kbase[KBASE_PARAM_PATH] + self.add_path = self._bool(self.kbase[KBASE_PARAM_PATH][0]) + self.enqueue_links = self._bool(self.kbase[KBASE_PARAM_ENQUEUE][0]) self.domain_regex = None if self.kbase[KBASE_PARAM_DOMAIN_REGEX][0]: @@ -136,7 +144,8 @@ def enqueue_link(self, fuzzresult, link_url, parsed_link): if not self.regex_param or ( self.regex_param and self.regex_param.search(new_link) is not None ): - self.queue_url(new_link) + if self.enqueue_links: + self.queue_url(new_link) self.add_verbose_result("link", "New link found", new_link) def from_domain(self, fuzzresult, parsed_link): From d21f7d77a203ab0e59d276cccfd1bf61ba36c78f Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 00:52:35 +0200 Subject: [PATCH 47/89] black format --- src/wfuzz/plugins/scripts/links.py | 7 +------ src/wfuzz/ui/console/clparser.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index 614fd634..aa46b2d7 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -30,12 +30,7 @@ class links(BasePlugin, DiscoveryPluginMixin): priority = 99 parameters = ( - ( - "enqueue", - "True", - False, - "If True, enqueue found links.", - ), + ("enqueue", "True", False, "If True, enqueue found links.",), ( "add_path", "False", diff --git a/src/wfuzz/ui/console/clparser.py b/src/wfuzz/ui/console/clparser.py index d5980c4d..392d07c1 100644 --- a/src/wfuzz/ui/console/clparser.py +++ b/src/wfuzz/ui/console/clparser.py @@ -130,12 +130,16 @@ def show_plugin_ext_help(self, registrant, category="$all$"): print(" %s" % desc_lines) print("Parameters:") for name, default_value, mandatory, description in plugin.parameters: - print(" {} {}{}: {}".format( - "+" if mandatory else "-", - name, - " (= %s)" % str(default_value) if default_value is not None else "", - description, - )) + print( + " {} {}{}: {}".format( + "+" if mandatory else "-", + name, + " (= %s)" % str(default_value) + if default_value is not None + else "", + description, + ) + ) print("\n") sys.exit(0) From 2f9dcc8173699f8cfbf8f110bed4e877b2b540cf Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 01:13:13 +0200 Subject: [PATCH 48/89] summary only in res __str__ --- src/wfuzz/fuzzobjects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index d3aff3fa..ae93f4a9 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -331,7 +331,8 @@ def __str__(self): self.description, ) for plugin in self.plugins_res: - res += "\n |_ %s" % plugin.issue + if plugin.itype == FuzzPlugin.SUMMARY_TYPE: + res += "\n |_ %s" % plugin.issue return res From 85d37533202f2e768c399a53549202cde1e34bf0 Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 01:53:45 +0200 Subject: [PATCH 49/89] test docker --- .github/workflows/docker-release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 222b1f0a..212180c0 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -1,8 +1,7 @@ name: docker-release on: - release: - types: [published] + push jobs: docker: @@ -32,5 +31,4 @@ jobs: file: Dockerfile push: true tags: | - ghcr.io/${{ github.repository_owner }}/wfuzz:${{ github.event.release.tag_name }} - ghcr.io/${{ github.repository_owner }}/wfuzz:latest + ghcr.io/${{ github.repository_owner }}/wfuzz:v3.0.3-beta From 1a65c9300527dd3907fdc0414f7c10b7f7f8f07e Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 01:57:28 +0200 Subject: [PATCH 50/89] test docker --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 212180c0..374be73a 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -31,4 +31,4 @@ jobs: file: Dockerfile push: true tags: | - ghcr.io/${{ github.repository_owner }}/wfuzz:v3.0.3-beta + ghcr.io/${{ github.repository_owner }}/wfuzz:v3.1.0-beta From ee253829502aab644b96dd02a75d8d67cd551825 Mon Sep 17 00:00:00 2001 From: javi Date: Sun, 25 Oct 2020 11:01:08 +0100 Subject: [PATCH 51/89] add jsparse regex to links --- src/wfuzz/plugins/scripts/links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index aa46b2d7..a261372b 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -58,9 +58,9 @@ def __init__(self): r'\b(?:(?', + r'', # http://en.wikipedia.org/wiki/Meta_refresh r'getJSON\("(.*?)"', + r"[^/][`'\"]([\/][a-zA-Z0-9_.-]+)+(?!(?:[,;\s]))", # based on https://github.com/nahamsec/JSParser/blob/master/handler.py#L93 ] self.regex = [] From 0fdfe2d3ffb93d10588a18e448fb49e248f487bd Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 27 Oct 2020 20:43:44 +0100 Subject: [PATCH 52/89] add plugin data in raw printer --- src/wfuzz/plugins/printers/printers.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/wfuzz/plugins/printers/printers.py b/src/wfuzz/plugins/printers/printers.py index 7552729b..003ee38a 100644 --- a/src/wfuzz/plugins/printers/printers.py +++ b/src/wfuzz/plugins/printers/printers.py @@ -285,8 +285,13 @@ def _print_verbose(self, res): ) ) - for i in res.plugins_res: - self.f.write(" |_ %s\n" % i.issue) + for plugin_res in res.plugins_res: + if plugin_res._verbose: + self.f.write( + " |_ {} {}\n".format( + plugin_res.issue, plugin_res.data if plugin_res.data else "" + ) + ) def _print(self, res): if res.exception: @@ -299,8 +304,13 @@ def _print(self, res): % (res.lines, res.words, res.chars, res.description) ) - for i in res.plugins_res: - self.f.write(" |_ %s\n" % i.issue) + for plugin_res in res.plugins_res: + if not plugin_res._verbose: + self.f.write( + " |_ {} {}\n".format( + plugin_res.issue, plugin_res.data if plugin_res.data else "" + ) + ) def result(self, res): if self.verbose: From 56f9add307bd0febd7404b7ec9191d227890978f Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 27 Oct 2020 21:20:45 +0100 Subject: [PATCH 53/89] field printer --- docs/user/basicusage.rst | 14 ++++++++++++-- src/wfuzz/plugins/printers/printers.py | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/user/basicusage.rst b/docs/user/basicusage.rst index 36e48146..69800bbe 100644 --- a/docs/user/basicusage.rst +++ b/docs/user/basicusage.rst @@ -252,7 +252,7 @@ For example, to show results in JSON format use the following command:: $ wfuzz -o json -w wordlist/general/common.txt http://testphp.vulnweb.com/FUZZ -When using the default output you can also select additional FuzzResult's fields to show, using --efield, together with the payload description:: +When using the default or raw output you can also select additional FuzzResult's fields to show, using --efield, together with the payload description:: $ wfuzz -z range --zD 0-1 -u http://testphp.vulnweb.com/artists.php?artist=FUZZ --efield r ... @@ -262,7 +262,7 @@ When using the default output you can also select additional FuzzResult's fields Host: testphp.vulnweb.com ... -The above is useful, for example, to debug what exact HTTP request Wfuzz sent to the remote Web server. +The above command is useful, for example, to debug what exact HTTP request Wfuzz sent to the remote Web server. To completely replace the default payload output you can use --field instead:: @@ -279,4 +279,14 @@ To completely replace the default payload output you can use --field instead:: 000000001: 200 104 L 364 W 4735 Ch "0 | http://testphp.vulnweb.com/artists.php?artist=0 | 4735" ... +The field printer can be used with a --efield or --field expression to list only the specified filter expressions without a header or footer:: + + + $ wfuzz -z list --zD https://www.airbnb.com/ --script=links --script-args=links.regex=.*js$,links.enqueue=False -u FUZZ -o field --field plugins.links.link | head -n3 + https://a0.muscache.com/airbnb/static/packages/4e8d-d5c346ee.js + https://a0.muscache.com/airbnb/static/packages/7afc-ac814a17.js + https://a0.muscache.com/airbnb/static/packages/7642-dcf4f8dc.js + +The above command is useful, for example, to pipe wfuzz into other tools or perform console scripts. + --efield and --field are in fact filter expressions. Check the filter language section in the advance usage document for the available fields and operators. diff --git a/src/wfuzz/plugins/printers/printers.py b/src/wfuzz/plugins/printers/printers.py index 003ee38a..065120f3 100644 --- a/src/wfuzz/plugins/printers/printers.py +++ b/src/wfuzz/plugins/printers/printers.py @@ -5,6 +5,7 @@ from wfuzz.externals.moduleman.plugin import moduleman_plugin from wfuzz.plugin_api.base import BasePrinter +from wfuzz.exception import FuzzExceptPluginBadParams @moduleman_plugin @@ -342,6 +343,31 @@ def footer(self, summary): ) +@moduleman_plugin +class field(BasePrinter): + name = "field" + author = ("Xavi Mendez (@xmendez)",) + version = "0.1" + summary = "Raw output format only showing the specified field expression. No header or footer." + category = ["default"] + priority = 99 + + def __init__(self, output): + BasePrinter.__init__(self, output) + + def header(self, summary): + pass + + def result(self, res): + if res._fields: + print(res._field("\n")) + else: + raise FuzzExceptPluginBadParams("You need to supply valid --field or --efield expression for unsing this printer.") + + def footer(self, summary): + pass + + @moduleman_plugin class csv(BasePrinter): name = "csv" From 078d332351ec286b5decfe9e5bea78e3421a606b Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 29 Oct 2020 01:01:15 +0100 Subject: [PATCH 54/89] revert workflow --- .github/workflows/docker-release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 374be73a..222b1f0a 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -1,7 +1,8 @@ name: docker-release on: - push + release: + types: [published] jobs: docker: @@ -31,4 +32,5 @@ jobs: file: Dockerfile push: true tags: | - ghcr.io/${{ github.repository_owner }}/wfuzz:v3.1.0-beta + ghcr.io/${{ github.repository_owner }}/wfuzz:${{ github.event.release.tag_name }} + ghcr.io/${{ github.repository_owner }}/wfuzz:latest From 9b821bd8ad3b154310083a27b811000f995a0906 Mon Sep 17 00:00:00 2001 From: javi Date: Thu, 29 Oct 2020 08:55:33 +0100 Subject: [PATCH 55/89] black formatting --- src/wfuzz/plugins/printers/printers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/printers/printers.py b/src/wfuzz/plugins/printers/printers.py index 065120f3..70b71ec6 100644 --- a/src/wfuzz/plugins/printers/printers.py +++ b/src/wfuzz/plugins/printers/printers.py @@ -362,7 +362,9 @@ def result(self, res): if res._fields: print(res._field("\n")) else: - raise FuzzExceptPluginBadParams("You need to supply valid --field or --efield expression for unsing this printer.") + raise FuzzExceptPluginBadParams( + "You need to supply valid --field or --efield expression for unsing this printer." + ) def footer(self, summary): pass From 57ad3f1cccca472e786fa24ef1abe771fb10f61f Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 18:05:05 +0100 Subject: [PATCH 56/89] verbose options --- src/wfuzz/options.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/wfuzz/options.py b/src/wfuzz/options.py index 9e96f174..44d8c610 100644 --- a/src/wfuzz/options.py +++ b/src/wfuzz/options.py @@ -386,6 +386,12 @@ def compile(self): self.http_pool = HttpPool(self) self.http_pool.register() + if self.data["colour"]: + Facade().printers.kbase["colour"] = True + + if self.data["verbose"]: + Facade().printers.kbase["verbose"] = True + return self def close(self): From 51c5b24ca2799c437e255c08a0dabbc716480245 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 20:25:41 +0100 Subject: [PATCH 57/89] is_visible in plugin --- src/wfuzz/factories/plugin_factory.py | 19 +++++++++++++++++-- src/wfuzz/fuzzobjects.py | 21 ++++++++++++++++----- src/wfuzz/fuzzqueues.py | 18 +++++------------- src/wfuzz/plugin_api/base.py | 13 +++---------- src/wfuzz/plugins/printers/printers.py | 4 ++-- src/wfuzz/plugins/scripts/headers.py | 6 +++--- src/wfuzz/plugins/scripts/links.py | 4 ++-- src/wfuzz/plugins/scripts/npm_deps.py | 4 ++-- src/wfuzz/ui/console/mvc.py | 4 +--- 9 files changed, 51 insertions(+), 42 deletions(-) diff --git a/src/wfuzz/factories/plugin_factory.py b/src/wfuzz/factories/plugin_factory.py index e8044a07..a660bea1 100644 --- a/src/wfuzz/factories/plugin_factory.py +++ b/src/wfuzz/factories/plugin_factory.py @@ -12,6 +12,7 @@ def __init__(self): "plugin_from_recursion": PluginRecursiveBuilder(), "plugin_from_error": PluginErrorBuilder(), "plugin_from_finding": PluginFindingBuilder(), + "plugin_from_summary": PluginFindingSummaryBuilder(), }, ) @@ -38,7 +39,7 @@ def __call__(self, name, exception): class PluginFindingBuilder: - def __call__(self, name, itype, message, data, verbose): + def __call__(self, name, itype, message, data, severity): plugin = FuzzPlugin() plugin.source = name plugin.issue = message @@ -46,7 +47,21 @@ def __call__(self, name, itype, message, data, verbose): plugin.data = data plugin._exception = None plugin._seed = None - plugin._verbose = verbose + plugin.severity = severity + + return plugin + + +class PluginFindingSummaryBuilder: + def __call__(self, message): + plugin = FuzzPlugin() + plugin.source = FuzzPlugin.OUTPUT_SOURCE + plugin.itype = FuzzPlugin.SUMMARY_ITYPE + plugin.severity = FuzzPlugin.NONE + plugin._exception = None + plugin.data = None + plugin._seed = None + plugin.issue = message return plugin diff --git a/src/wfuzz/fuzzobjects.py b/src/wfuzz/fuzzobjects.py index ae93f4a9..fbd2a526 100644 --- a/src/wfuzz/fuzzobjects.py +++ b/src/wfuzz/fuzzobjects.py @@ -293,7 +293,7 @@ def plugins(self): dic = defaultdict(lambda: defaultdict(list)) for pl in self.plugins_res: - if pl.source == FuzzPlugin.OUTPUT_CAT: + if pl.source == FuzzPlugin.OUTPUT_SOURCE: continue dic[pl.source][pl.itype].append(pl.data) @@ -331,7 +331,7 @@ def __str__(self): self.description, ) for plugin in self.plugins_res: - if plugin.itype == FuzzPlugin.SUMMARY_TYPE: + if plugin.itype == FuzzPlugin.SUMMARY_ITYPE: res += "\n |_ %s" % plugin.issue return res @@ -402,8 +402,10 @@ def update_from_options(self, options): class FuzzPlugin(FuzzItem): - OUTPUT_CAT = "output" - SUMMARY_TYPE = "summary" + OUTPUT_SOURCE = "output" + SUMMARY_ITYPE = "summary" + NONE, INFO, LOW, MEDIUM, HIGH, CRITICAL = range(6) + MIN_VERBOSE = INFO def __init__(self): FuzzItem.__init__(self, FuzzType.PLUGIN) @@ -413,4 +415,13 @@ def __init__(self): self.data = "" self._exception = None self._seed = None - self._verbose = False + self.severity = self.INFO + + def is_visible(self, verbose): + if verbose and self.itype == self.SUMMARY_ITYPE: + return False + + if not verbose and self.severity >= self.MIN_VERBOSE: + return False + + return True diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index acfe2786..3ca9917b 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -8,7 +8,7 @@ from .factories.fuzzresfactory import resfactory from .factories.plugin_factory import plugin_factory from .factories.payman import payman_factory -from .fuzzobjects import FuzzPlugin, FuzzType, FuzzItem, FuzzWord, FuzzWordType +from .fuzzobjects import FuzzType, FuzzItem, FuzzWord, FuzzWordType from .myqueues import FuzzQueue from .exception import ( FuzzExceptInternalError, @@ -362,14 +362,10 @@ def process_results(self, res, plugins_res_queue): for domain, enq_num in plugin_type.items(): res.plugins_res.append( plugin_factory.create( - "plugin_from_finding", - FuzzPlugin.OUTPUT_CAT, - FuzzPlugin.SUMMARY_TYPE, + "plugin_from_summary", "Plugin {}: {} new {}(s) found.".format( plugin_name, enq_num, domain - ), - None, - False, + ) ) ) @@ -394,12 +390,8 @@ def process(self, fuzz_res): fuzz_res.plugins_res.append( plugin_factory.create( - "plugin_from_finding", - FuzzPlugin.OUTPUT_CAT, - FuzzPlugin.SUMMARY_TYPE, - "Enqueued response for recursion (level=%d)" % (seed.rlevel), - None, - False, + "plugin_from_summary", + "Enqueued response for recursion (level=%d)" % (seed.rlevel) ) ) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index 4ac05e2f..5d7855d3 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -1,4 +1,4 @@ -from wfuzz.fuzzobjects import FuzzWord +from wfuzz.fuzzobjects import FuzzWord, FuzzPlugin from wfuzz.exception import ( FuzzExceptBadFile, FuzzExceptBadOptions, @@ -59,17 +59,10 @@ def process(self, fuzzresult): def validate(self): raise FuzzExceptPluginError("Method count not implemented") - def add_result(self, itype, issue, data): + def add_result(self, itype, issue, data, severity=FuzzPlugin.INFO): self.results_queue.put( plugin_factory.create( - "plugin_from_finding", self.name, itype, issue, data, False - ) - ) - - def add_verbose_result(self, itype, issue, data): - self.results_queue.put( - plugin_factory.create( - "plugin_from_finding", self.name, itype, issue, data, True + "plugin_from_finding", self.name, itype, issue, data, severity ) ) diff --git a/src/wfuzz/plugins/printers/printers.py b/src/wfuzz/plugins/printers/printers.py index 70b71ec6..e7572053 100644 --- a/src/wfuzz/plugins/printers/printers.py +++ b/src/wfuzz/plugins/printers/printers.py @@ -287,7 +287,7 @@ def _print_verbose(self, res): ) for plugin_res in res.plugins_res: - if plugin_res._verbose: + if plugin_res.is_visible(self.verbose): self.f.write( " |_ {} {}\n".format( plugin_res.issue, plugin_res.data if plugin_res.data else "" @@ -306,7 +306,7 @@ def _print(self, res): ) for plugin_res in res.plugins_res: - if not plugin_res._verbose: + if plugin_res.is_visible(self.verbose): self.f.write( " |_ {} {}\n".format( plugin_res.issue, plugin_res.data if plugin_res.data else "" diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index 94b30e30..eee3584a 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -129,7 +129,7 @@ def check_request_header(self, fuzzresult, header, value): header_value.lower() not in self.kbase[KBASE_KEY_REQ_UNCOMMON] or KBASE_KEY_REQ_UNCOMMON not in self.kbase ): - self.add_verbose_result( + self.add_result( "reqheader", "New uncommon HTTP request header", "{}: {}".format(header_value, value), @@ -147,7 +147,7 @@ def check_response_header(self, fuzzresult, header, value): header_value.lower() not in self.kbase[KBASE_KEY_RESP_UNCOMMON] or KBASE_KEY_RESP_UNCOMMON not in self.kbase ): - self.add_verbose_result( + self.add_result( "header", "New uncommon HTTP response header", "{}: {}".format( @@ -163,7 +163,7 @@ def check_server_header(self, fuzzresult, header, value): value.lower() not in self.kbase[KBASE_KEY] or KBASE_KEY not in self.kbase ): - self.add_verbose_result( + self.add_result( "server", "New server HTTP response header", "{}: {}".format(header, value), diff --git a/src/wfuzz/plugins/scripts/links.py b/src/wfuzz/plugins/scripts/links.py index a261372b..e0d0cd76 100644 --- a/src/wfuzz/plugins/scripts/links.py +++ b/src/wfuzz/plugins/scripts/links.py @@ -141,7 +141,7 @@ def enqueue_link(self, fuzzresult, link_url, parsed_link): ): if self.enqueue_links: self.queue_url(new_link) - self.add_verbose_result("link", "New link found", new_link) + self.add_result("link", "New link found", new_link) def from_domain(self, fuzzresult, parsed_link): # relative path @@ -164,6 +164,6 @@ def from_domain(self, fuzzresult, parsed_link): and parsed_link.netloc not in self.kbase[KBASE_NEW_DOMAIN] ): self.kbase[KBASE_NEW_DOMAIN].append(parsed_link.netloc) - self.add_verbose_result( + self.add_result( "domain", "New domain found (link not enqueued)", parsed_link.netloc ) diff --git a/src/wfuzz/plugins/scripts/npm_deps.py b/src/wfuzz/plugins/scripts/npm_deps.py index 10fda31a..dde3156a 100644 --- a/src/wfuzz/plugins/scripts/npm_deps.py +++ b/src/wfuzz/plugins/scripts/npm_deps.py @@ -42,8 +42,8 @@ def validate(self, fuzzresult): def process(self, fuzzresult): if self.match_dev: for name, version in self.REGEX_PATT.findall(self.match_dev.group(1)): - self.add_verbose_result("dependency", "npm dependency", name) + self.add_result("dependency", "npm dependency", name) if self.match: for name, version in self.REGEX_PATT.findall(self.match.group(1)): - self.add_verbose_result("dev_dependency", "npm dev dependency", name) + self.add_result("dev_dependency", "npm dev dependency", name) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 51b79f91..5aa0d70f 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -322,9 +322,7 @@ def result(self, res): if res.plugins_res: for plugin_res in res.plugins_res: - if (plugin_res._verbose and not self.verbose) or ( - plugin_res.itype == FuzzPlugin.SUMMARY_TYPE and self.verbose - ): + if not plugin_res.is_visible(self.verbose): continue sys.stdout.write( From b6a776ed11ede5d5930eb7ec1f25d5615412e0df Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 20:27:54 +0100 Subject: [PATCH 58/89] black format --- src/wfuzz/fuzzqueues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index 3ca9917b..d69f3bf8 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -365,7 +365,7 @@ def process_results(self, res, plugins_res_queue): "plugin_from_summary", "Plugin {}: {} new {}(s) found.".format( plugin_name, enq_num, domain - ) + ), ) ) @@ -391,7 +391,7 @@ def process(self, fuzz_res): fuzz_res.plugins_res.append( plugin_factory.create( "plugin_from_summary", - "Enqueued response for recursion (level=%d)" % (seed.rlevel) + "Enqueued response for recursion (level=%d)" % (seed.rlevel), ) ) From 0cad4b4a06e999f40eee8cbf0e23a1c979d8698b Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 20:43:01 +0100 Subject: [PATCH 59/89] reinstalling pycurl instructions --- docs/user/installation.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/user/installation.rst b/docs/user/installation.rst index c21f6566..9e04a7e7 100644 --- a/docs/user/installation.rst +++ b/docs/user/installation.rst @@ -91,6 +91,16 @@ If you get errors such as:: Run brew update && brew upgrade +If you get an error such as:: + + ImportError: pycurl: libcurl link-time ssl backends (secure-transport, openssl) do not include compile-time ssl backend (none/other) + +That might indicate that pycurl was reinstalled and not linked to the SSL correctly. Uninstall pycurl as follows:: + + $ pip uninstall pycurl + +and re-install pycurl starting from step 4 above. + Pycurl on Windows ----------------- From 82458754002f3f01783e8db3194117811c86e783 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 21:00:23 +0100 Subject: [PATCH 60/89] fix test --- tests/plugins/test_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/test_links.py b/tests/plugins/test_links.py index 553fbfe8..388277f5 100644 --- a/tests/plugins/test_links.py +++ b/tests/plugins/test_links.py @@ -7,7 +7,7 @@ @pytest.mark.parametrize( "example_full_fuzzres_content, expected_links", [ - (b'\n', [],), + # getting data-href for now (b'\n', [],), ( b'\n', ["http://www.wfuzz.org/1.json", "http://www.wfuzz.org/2.json"], From a12472460aa3bc62c6f00c019032ec0f16381bba Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 21:33:44 +0100 Subject: [PATCH 61/89] test for is_visible --- tests/plugins/test_summary.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/plugins/test_summary.py diff --git a/tests/plugins/test_summary.py b/tests/plugins/test_summary.py new file mode 100644 index 00000000..19ea27be --- /dev/null +++ b/tests/plugins/test_summary.py @@ -0,0 +1,36 @@ +from wfuzz.factories.plugin_factory import plugin_factory +from wfuzz.fuzzobjects import FuzzPlugin + +from queue import Queue + + +def test_sum_plugin_output(example_full_fuzzres): + plugin = plugin_factory.create("plugin_from_summary", "a message") + + assert plugin.is_visible(True) is False + assert plugin.is_visible(False) is True + + +def test_find_plugin_output_from_factory(): + plugin = plugin_factory.create( + "plugin_from_finding", + "a plugin", + "a source", + "an issue", + "some data", + FuzzPlugin.INFO, + ) + + assert plugin.is_visible(True) is True + assert plugin.is_visible(False) is False + + +def test_find_plugin_output(get_plugin): + plugin = get_plugin("links")[0] + plugin.results_queue = Queue() + plugin.add_result("a source", "an issue", "some data", FuzzPlugin.INFO) + + plugin_res = plugin.results_queue.get() + + assert plugin_res.is_visible(True) is True + assert plugin_res.is_visible(False) is False From b6429c4144b4bf1ab96211f083f6f7917f3e24a0 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 21:39:03 +0100 Subject: [PATCH 62/89] fix latest release link --- docs/user/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/installation.rst b/docs/user/installation.rst index 9e04a7e7..ce7069ec 100644 --- a/docs/user/installation.rst +++ b/docs/user/installation.rst @@ -26,7 +26,7 @@ You can either clone the public repository:: $ git clone git://github.com/xmendez/wfuzz.git -Or download last `release _`. +Or download last `release `_. Once you have a copy of the source, you can embed it in your own Python package, or install it into your site-packages easily:: From 5f045956e9e30bda8896896d3e119c5accad2626 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 23:34:09 +0100 Subject: [PATCH 63/89] change enqueue message --- src/wfuzz/fuzzqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/fuzzqueues.py b/src/wfuzz/fuzzqueues.py index d69f3bf8..f710aba6 100644 --- a/src/wfuzz/fuzzqueues.py +++ b/src/wfuzz/fuzzqueues.py @@ -353,7 +353,7 @@ def process_results(self, res, plugins_res_queue): self.stats.backfeed.inc() self.stats.pending_fuzz.inc() self.send(item._seed) - enq_item[item.source]["request"] += 1 + enq_item[item.source]["request enqueued"] += 1 elif item.issue: enq_item[item.source][item.itype] += 1 res.plugins_res.append(item) From 39db47703ffcda5a22017298dc27d18c2ee12470 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 23:42:44 +0100 Subject: [PATCH 64/89] add result in robots --- src/wfuzz/plugins/scripts/robots.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/scripts/robots.py b/src/wfuzz/plugins/scripts/robots.py index 037c9aea..4ca83e56 100644 --- a/src/wfuzz/plugins/scripts/robots.py +++ b/src/wfuzz/plugins/scripts/robots.py @@ -53,4 +53,6 @@ def process(self, fuzzresult): url = url.strip(" *") if url: - self.queue_url(urljoin(fuzzresult.url, url)) + new_link = urljoin(fuzzresult.url, url) + self.queue_url(new_link) + self.add_result("link", "New link found", new_link) From 398086154e3a31ebffa8619ca535cfa7704a2f0e Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sat, 31 Oct 2020 23:53:14 +0100 Subject: [PATCH 65/89] use python3 notation --- src/wfuzz/plugin_api/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index 5d7855d3..38457be6 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -24,7 +24,7 @@ def __init__(self): # check mandatory params, assign default values for name, default_value, required, description in self.parameters: - param_name = "%s.%s" % (self.name, name) + param_name = f"{self.name}.{name}" if required and param_name not in list(self.kbase.keys()): raise FuzzExceptBadOptions( From 52b649e0f2c29c90911f5065412931d766c5bda1 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sun, 1 Nov 2020 00:06:54 +0100 Subject: [PATCH 66/89] only add server value --- src/wfuzz/plugins/scripts/headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index eee3584a..082a2e22 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -166,7 +166,7 @@ def check_server_header(self, fuzzresult, header, value): self.add_result( "server", "New server HTTP response header", - "{}: {}".format(header, value), + "{}".format(value), ) self.kbase[KBASE_KEY].append(value.lower()) From 6e791e4cca7133e8bc3a97170d1d9638bc30709f Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sun, 1 Nov 2020 00:09:18 +0100 Subject: [PATCH 67/89] only print if result --- src/wfuzz/plugins/printers/printers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wfuzz/plugins/printers/printers.py b/src/wfuzz/plugins/printers/printers.py index e7572053..6677cffe 100644 --- a/src/wfuzz/plugins/printers/printers.py +++ b/src/wfuzz/plugins/printers/printers.py @@ -360,7 +360,9 @@ def header(self, summary): def result(self, res): if res._fields: - print(res._field("\n")) + to_print = res._field("\n") + if to_print: + print(to_print) else: raise FuzzExceptPluginBadParams( "You need to supply valid --field or --efield expression for unsing this printer." From 048130137ccfb4acdef4384761696ae01404f2a9 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sun, 1 Nov 2020 01:26:09 +0100 Subject: [PATCH 68/89] handle exceptions in wfencode --- src/wfuzz/wfuzz.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index eee2aac2..10cab724 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -216,6 +216,8 @@ def usage(): ) except FuzzException as e: warnings.warn(("\nFatal exception: %s" % str(e))) + except Exception as e: + warnings.warn(("Unhandled exception: %s" % str(e))) def main_gui(): From 4cc75e280e37885bfe8d4603f4207cc23564287f Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sun, 1 Nov 2020 10:47:43 +0100 Subject: [PATCH 69/89] wfencode stdin --- src/wfuzz/wfuzz.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/wfuzz/wfuzz.py b/src/wfuzz/wfuzz.py index 10cab724..559b9df6 100644 --- a/src/wfuzz/wfuzz.py +++ b/src/wfuzz/wfuzz.py @@ -177,28 +177,39 @@ def usage(): print("\n\twfencode --help This help") print("\twfencode -d decoder_name string_to_decode") print("\twfencode -e encoder_name string_to_encode") + print("\twfencode -e encoder_name -i <>") print() from .api import encode, decode import getopt try: - opts, args = getopt.getopt(sys.argv[1:], "he:d:", ["help"]) + opts, args = getopt.getopt(sys.argv[1:], "hie:d:", ["help"]) except getopt.GetoptError as err: warnings.warn(str(err)) usage() sys.exit(2) - if len(args) == 0: + arg_keys = [i for i, j in opts] + + if len(args) == 0 and "-i" not in arg_keys: usage() sys.exit() try: for o, value in opts: if o == "-e": - print((encode(value, args[0]))) + if "-i" in arg_keys: + for std in sys.stdin: + print(encode(value, std.strip())) + else: + print(encode(value, args[0])) elif o == "-d": - print((decode(value, args[0]))) + if "-i" in arg_keys: + for std in sys.stdin: + print(decode(value, std.strip())) + else: + print(decode(value, args[0])) elif o in ("-h", "--help"): usage() sys.exit() From 7ee1aa0cee33cbf96e4b83b9b6896ecf4005195b Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Sun, 1 Nov 2020 12:56:16 +0100 Subject: [PATCH 70/89] change plugins filter doc --- docs/user/advanced.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index f17dd608..558c4868 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -482,7 +482,7 @@ lines l Wfuzz's result HTTP response lines words w Wfuzz's result HTTP response words md5 Wfuzz's result HTTP response md5 hash history r Wfuzz's result associated FuzzRequest object -plugins Wfuzz's results associated plugins result in the form of {'plugin id': ['result']} +plugins Wfuzz's plugins scan results ============ ============== ============================================= FuzzRequest object's attribute (you need to use the r. prefix) such as: @@ -740,6 +740,12 @@ The above command will generate HTTP requests such as the following:: You can filter the payload using the filter grammar as described before. +Reutilising previous results +-------------------------------------- + +Plugins results contain a treasure trove of data. Wfuzz payloads and object introspection (explained in the filter grammar section) exposes a Python object interface to plugins results. +This allows you to perform semi-automatic tests based on plugins results or compile a set of results to be used in another tool. + Request mangling ^^^^^^^^^ From d2e0d8bbad27c94bf8374ed42a12aec0e2e31cfc Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 14:32:42 +0100 Subject: [PATCH 71/89] burplog test and fix --- src/wfuzz/plugins/payloads/burplog.py | 2 +- tests/plugins/test_burplog.py | 132 ++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 tests/plugins/test_burplog.py diff --git a/src/wfuzz/plugins/payloads/burplog.py b/src/wfuzz/plugins/payloads/burplog.py index 4ece4f20..9664e96f 100644 --- a/src/wfuzz/plugins/payloads/burplog.py +++ b/src/wfuzz/plugins/payloads/burplog.py @@ -113,7 +113,7 @@ def parse_burp_log(self, burp_log): if rl == CRLF: fr = FuzzRequest() fr.update_from_raw_http( - raw_request, host[: host.find("://")], raw_response + raw_request, host[: host.find("://")], raw_response.rstrip() ) frr = FuzzResult(history=fr) diff --git a/tests/plugins/test_burplog.py b/tests/plugins/test_burplog.py new file mode 100644 index 00000000..b8febb6c --- /dev/null +++ b/tests/plugins/test_burplog.py @@ -0,0 +1,132 @@ +import pytest +import sys +from io import BytesIO + +import wfuzz +from wfuzz.facade import Facade + +try: + # Python >= 3.3 + from unittest import mock +except ImportError: + # Python < 3.3 + import mock + + +@pytest.fixture +def burp_log_raw(): + return """====================================================== +22:35:55 https://aus5.mozilla.org:443 [35.244.181.201] +====================================================== +GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1 +Host: aus5.mozilla.org +User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0 +Accept: */* +Accept-Language: en-GB,en;q=0.5 +Accept-Encoding: gzip, deflate +Cache-Control: no-cache +Pragma: no-cache +Connection: close + + +====================================================== +HTTP/1.1 200 OK +Server: nginx/1.17.9 +Date: Sun, 01 Nov 2020 21:35:08 GMT +Content-Type: text/xml; charset=utf-8 +Content-Length: 42 +Strict-Transport-Security: max-age=31536000; +X-Content-Type-Options: nosniff +Content-Security-Policy: default-src 'none'; frame-ancestors 'none' +X-Proxy-Cache-Status: EXPIRED +Via: 1.1 google +Age: 47 +Cache-Control: public, max-age=90 +Alt-Svc: clear +Connection: close + + + + +====================================================== + + + +""" + + +class mock_saved_session(object): + def __init__(self, infile): + self.outfile = BytesIO(bytes(infile, "ascii")) + self.outfile.seek(0) + self.outfile.name = "mockfile" + + def close(self): + pass + + def read(self, *args, **kwargs): + return self.outfile.read(*args, **kwargs) + + def seek(self, *args, **kwargs): + return self.outfile.seek(*args, **kwargs) + + def tell(self): + return self.outfile.tell() + + def readline(self, *args, **kwargs): + line = self.outfile.readline() + if line: + return line.decode("utf-8") + return "" + + +def test_burplog(burp_log_raw): + # load plugins before mocking file object + Facade().payloads + + m = mock.MagicMock(name="open", spec=open) + m.return_value = mock_saved_session(burp_log_raw) + + mocked_fun = "builtins.open" if sys.version_info >= (3, 0) else "__builtin__.open" + with mock.patch(mocked_fun, m, create=True): + payload_list = list( + wfuzz.payload( + **{ + "payloads": [ + ("burplog", {"default": "mockedfile", "encoder": None}, None) + ], + } + ) + ) + + fres = payload_list[0][0] + + assert fres.history.headers.response["Server"] == "nginx/1.17.9" + assert fres.history.headers.response["server"] == "nginx/1.17.9" + assert fres.history.content == '\n\n' + assert fres.history.headers.request == { + "Host": "aus5.mozilla.org", + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0", + "Accept": "*/*", + "Accept-Language": "en-GB,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "close", + } + + assert fres.history.headers.response == { + 'Server': 'nginx/1.17.9', + 'Date': 'Sun, 01 Nov 2020 21:35:08 GMT', + 'Content-Type': 'text/xml; charset=utf-8', + 'Content-Length': '42', + 'Strict-Transport-Security': 'max-age=31536000;', + 'X-Content-Type-Options': 'nosniff', + 'Content-Security-Policy': "default-src 'none'; frame-ancestors 'none'", + 'X-Proxy-Cache-Status': 'EXPIRED', + 'Via': '1.1 google', + 'Age': '47', + 'Cache-Control': 'public, max-age=90', + 'Alt-Svc': 'clear', + 'Connection': 'close', + } From d65e49bc2671fcc8ecda28b4001f6acf6c854a3b Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 15:58:22 +0100 Subject: [PATCH 72/89] burplog test and fix --- src/wfuzz/plugins/payloads/burplog.py | 3 +- tests/plugins/test_burplog.py | 350 +++++++++++++++++++------- 2 files changed, 255 insertions(+), 98 deletions(-) diff --git a/src/wfuzz/plugins/payloads/burplog.py b/src/wfuzz/plugins/payloads/burplog.py index 9664e96f..7dcd0f8a 100644 --- a/src/wfuzz/plugins/payloads/burplog.py +++ b/src/wfuzz/plugins/payloads/burplog.py @@ -112,8 +112,9 @@ def parse_burp_log(self, burp_log): elif history == "DELIM4": if rl == CRLF: fr = FuzzRequest() + # last read line contains an extra CRLF fr.update_from_raw_http( - raw_request, host[: host.find("://")], raw_response.rstrip() + raw_request, host[: host.find("://")], raw_response[:-1] ) frr = FuzzResult(history=fr) diff --git a/tests/plugins/test_burplog.py b/tests/plugins/test_burplog.py index b8febb6c..162005e8 100644 --- a/tests/plugins/test_burplog.py +++ b/tests/plugins/test_burplog.py @@ -14,78 +14,261 @@ @pytest.fixture -def burp_log_raw(): - return """====================================================== -22:35:55 https://aus5.mozilla.org:443 [35.244.181.201] -====================================================== -GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1 -Host: aus5.mozilla.org -User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0 -Accept: */* -Accept-Language: en-GB,en;q=0.5 -Accept-Encoding: gzip, deflate -Cache-Control: no-cache -Pragma: no-cache -Connection: close - - -====================================================== -HTTP/1.1 200 OK -Server: nginx/1.17.9 -Date: Sun, 01 Nov 2020 21:35:08 GMT -Content-Type: text/xml; charset=utf-8 -Content-Length: 42 -Strict-Transport-Security: max-age=31536000; -X-Content-Type-Options: nosniff -Content-Security-Policy: default-src 'none'; frame-ancestors 'none' -X-Proxy-Cache-Status: EXPIRED -Via: 1.1 google -Age: 47 -Cache-Control: public, max-age=90 -Alt-Svc: clear -Connection: close - - - - -====================================================== - - - -""" - - -class mock_saved_session(object): - def __init__(self, infile): - self.outfile = BytesIO(bytes(infile, "ascii")) - self.outfile.seek(0) - self.outfile.name = "mockfile" - - def close(self): - pass - - def read(self, *args, **kwargs): - return self.outfile.read(*args, **kwargs) - - def seek(self, *args, **kwargs): - return self.outfile.seek(*args, **kwargs) - - def tell(self): - return self.outfile.tell() - - def readline(self, *args, **kwargs): - line = self.outfile.readline() - if line: - return line.decode("utf-8") - return "" - - -def test_burplog(burp_log_raw): +def burplog_file(request): + class mock_saved_session(object): + def __init__(self, infile): + self.outfile = BytesIO(bytes(infile, "ascii")) + self.outfile.seek(0) + self.outfile.name = "mockfile" + + def close(self): + pass + + def read(self, *args, **kwargs): + return self.outfile.read(*args, **kwargs) + + def seek(self, *args, **kwargs): + return self.outfile.seek(*args, **kwargs) + + def tell(self): + return self.outfile.tell() + + def readline(self, *args, **kwargs): + line = self.outfile.readline() + if line: + return line.decode("utf-8") + return "" + + return mock_saved_session(request.param) + + +@pytest.mark.parametrize( + "burplog_file, expected_content", + [ + # ( + # ( + # "======================================================\n" + # "22:35:55 https://aus5.mozilla.org:443 [35.244.181.201]\n" + # "======================================================\n" + # "GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1\n" + # "Host: aus5.mozilla.org\n" + # "\n" + # "\n" + # "======================================================\n" + # "HTTP/1.1 200 OK\n" + # "Server: nginx/1.17.9\n" + # "\n" + # "\n" + # "\n" + # "\r\n" + # "======================================================\n" + # "\n" + # "\n" + # "\n" + # ), + # '\n\n', + # ), + ( + ( + "======================================================\n" + "22:35:55 https://aus5.mozilla.org:443 [35.244.181.201]\n" + "======================================================\n" + "GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1\n" + "Host: aus5.mozilla.org\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 200 OK\n" + "Server: nginx/1.17.9\n" + "\n" + "\n" + "\n" + " \n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + '\n\n ', + ), + ( + ( + "======================================================\n" + "22:35:55 https://aus5.mozilla.org:443 [35.244.181.201]\n" + "======================================================\n" + "GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1\n" + "Host: aus5.mozilla.org\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 200 OK\n" + "Server: nginx/1.17.9\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + '\n\n\n', + ), + ( + ( + "======================================================\n" + "22:35:55 https://aus5.mozilla.org:443 [35.244.181.201]\n" + "======================================================\n" + "GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1\n" + "Host: aus5.mozilla.org\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 200 OK\n" + "Server: nginx/1.17.9\n" + "\n" + "\n" + "\n" + "\n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + '\n\n', + ), + ( + ( + "======================================================\n" + "22:26:48 http://testphp.vulnweb.com:80 [176.28.50.165]\n" + "======================================================\n" + "GET /style.css HTTP/1.1\n" + "Host: testphp.vulnweb.com\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 304 Not Modified\n" + "Server: nginx/1.4.1\n" + "Date: Mon, 19 Jan 1970 15:36:40 GMT\n" + "Last-Modified: Wed, 11 May 2011 10:27:48 GMT\n" + "Connection: close\n" + "ETag: \"4dca64a4-156a\"\n" + "\n" + "\n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + '', + ), + ], + indirect=["burplog_file"], +) +def test_burplog_content(burplog_file, expected_content): + # load plugins before mocking file object + Facade().payloads + + m = mock.MagicMock(name="open", spec=open) + m.return_value = burplog_file + + mocked_fun = "builtins.open" if sys.version_info >= (3, 0) else "__builtin__.open" + with mock.patch(mocked_fun, m, create=True): + payload_list = list( + wfuzz.payload( + **{ + "payloads": [ + ("burplog", {"default": "mockedfile", "encoder": None}, None) + ], + } + ) + ) + + fres = payload_list[0][0] + + assert fres.history.content == expected_content + + +@pytest.mark.parametrize( + "burplog_file, expected_req_headers, expected_resp_headers", + [ + ( + ( + "======================================================\n" + "22:35:55 https://aus5.mozilla.org:443 [35.244.181.201]\n" + "======================================================\n" + "GET /update/3/SystemAddons/81.0/20200917005511/Linux_x86_64-gcc3/null/release-cck-ubuntu/Linux%205.4.0-48-generic%20(GTK%203.24.20%2Clibpulse%2013.99.0)/canonical/1.0/update.xml HTTP/1.1\n" + "Host: aus5.mozilla.org\n" + "User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0\n" + "Accept: */*\n" + "Accept-Language: en-GB,en;q=0.5\n" + "Accept-Encoding: gzip, deflate\n" + "Cache-Control: no-cache\n" + "Pragma: no-cache\n" + "Connection: close\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 200 OK\n" + "Server: nginx/1.17.9\n" + "Date: Sun, 01 Nov 2020 21:35:08 GMT\n" + "Content-Type: text/xml; charset=utf-8\n" + "Content-Length: 42\n" + "Strict-Transport-Security: max-age=31536000;\n" + "X-Content-Type-Options: nosniff\n" + "Content-Security-Policy: default-src 'none'; frame-ancestors 'none'\n" + "X-Proxy-Cache-Status: EXPIRED\n" + "Via: 1.1 google\n" + "Age: 47\n" + "Cache-Control: public, max-age=90\n" + "Alt-Svc: clear\n" + "Connection: close\n" + "\n" + "\n" + "\n" + "\n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + { + "Host": "aus5.mozilla.org", + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0", + "Accept": "*/*", + "Accept-Language": "en-GB,en;q=0.5", + "Accept-Encoding": "gzip, deflate", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "close", + }, + { + 'Server': 'nginx/1.17.9', + 'Date': 'Sun, 01 Nov 2020 21:35:08 GMT', + 'Content-Type': 'text/xml; charset=utf-8', + 'Content-Length': '42', + 'Strict-Transport-Security': 'max-age=31536000;', + 'X-Content-Type-Options': 'nosniff', + 'Content-Security-Policy': "default-src 'none'; frame-ancestors 'none'", + 'X-Proxy-Cache-Status': 'EXPIRED', + 'Via': '1.1 google', + 'Age': '47', + 'Cache-Control': 'public, max-age=90', + 'Alt-Svc': 'clear', + 'Connection': 'close', + } + + ), + ], + indirect=["burplog_file"], +) +def test_burplog_headers(burplog_file, expected_req_headers, expected_resp_headers): # load plugins before mocking file object Facade().payloads m = mock.MagicMock(name="open", spec=open) - m.return_value = mock_saved_session(burp_log_raw) + m.return_value = burplog_file mocked_fun = "builtins.open" if sys.version_info >= (3, 0) else "__builtin__.open" with mock.patch(mocked_fun, m, create=True): @@ -101,32 +284,5 @@ def test_burplog(burp_log_raw): fres = payload_list[0][0] - assert fres.history.headers.response["Server"] == "nginx/1.17.9" - assert fres.history.headers.response["server"] == "nginx/1.17.9" - assert fres.history.content == '\n\n' - assert fres.history.headers.request == { - "Host": "aus5.mozilla.org", - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0", - "Accept": "*/*", - "Accept-Language": "en-GB,en;q=0.5", - "Accept-Encoding": "gzip, deflate", - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "Connection": "close", - } - - assert fres.history.headers.response == { - 'Server': 'nginx/1.17.9', - 'Date': 'Sun, 01 Nov 2020 21:35:08 GMT', - 'Content-Type': 'text/xml; charset=utf-8', - 'Content-Length': '42', - 'Strict-Transport-Security': 'max-age=31536000;', - 'X-Content-Type-Options': 'nosniff', - 'Content-Security-Policy': "default-src 'none'; frame-ancestors 'none'", - 'X-Proxy-Cache-Status': 'EXPIRED', - 'Via': '1.1 google', - 'Age': '47', - 'Cache-Control': 'public, max-age=90', - 'Alt-Svc': 'clear', - 'Connection': 'close', - } + assert fres.history.headers.request == expected_req_headers + assert fres.history.headers.response == expected_resp_headers From 6b3bf8c3840f42c2a991678c9efdd490aa08379a Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 19:41:13 +0100 Subject: [PATCH 73/89] formatting --- src/wfuzz/plugins/scripts/headers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/wfuzz/plugins/scripts/headers.py b/src/wfuzz/plugins/scripts/headers.py index 082a2e22..661bef82 100644 --- a/src/wfuzz/plugins/scripts/headers.py +++ b/src/wfuzz/plugins/scripts/headers.py @@ -164,9 +164,7 @@ def check_server_header(self, fuzzresult, header, value): or KBASE_KEY not in self.kbase ): self.add_result( - "server", - "New server HTTP response header", - "{}".format(value), + "server", "New server HTTP response header", "{}".format(value), ) self.kbase[KBASE_KEY].append(value.lower()) From 353f01e647270f8c130debc31dfd569085874b19 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 20:04:12 +0100 Subject: [PATCH 74/89] fix duplicated test --- tests/test_acceptance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index a594a66b..c453297b 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -118,7 +118,7 @@ ), # set values ( - "test_desc_concat_number", + "test_desc_concat_number_slice", "-z range,1-1 {}/FUZZ".format(HTTPBIN_URL), "-z wfuzzp,$$PREVFILE$$ --slice r.c:=302 FUZZ[url]FUZZ[c]", ["http://localhost:9000/1 - 302"], From 2b547b9405fe007224e9e77a7a4bb4ebf4d7c45d Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 20:44:15 +0100 Subject: [PATCH 75/89] add tests and tox action --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 60370f7b..4a8f6c1f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ .PHONY: docs -test: +tox: pip install tox tox --recreate +test: + pytest -v -s tests/ flake8: black --check src tests flake8 src tests From 7d5208708984ee3c6f16f2e43572decf15aeb42b Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 20:49:38 +0100 Subject: [PATCH 76/89] fix --slice when using fuzzres payload --- docs/user/advanced.rst | 31 +++++++++++++++++++++++++++++++ src/wfuzz/dictionaries.py | 17 +++++++++++++---- tests/test_acceptance.py | 2 +- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 558c4868..d273db33 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -633,6 +633,37 @@ The specified expression must return a boolean value, an example, using the uniq It is worth noting that, the type of payload dictates the available language symbols. For example, a dictionary payload such as in the example above does not have a full FuzzResult object context and therefore object fields cannot be used. +When slicing a FuzzResult payload, you are accessing the FuzzResult directly, therefore given a previous session such as:: + + $ wfuzz -z range --zD 0-0 -u http://www.google.com/FUZZ --oF /tmp/test1 + ... + 000000001: 404 11 L 72 W 1558 Ch "0" + ... + +this can be used to filter the payload:: + + $ wfpayload -z wfuzzp --zD /tmp/test1 --slice "c=404" + ... + 000000001: 404 11 L 72 W 1558 Ch "0" + ... + + $ wfpayload -z wfuzzp --zD /tmp/test1 --slice "c!=404" + ... + wfuzz.py:168: UserWarning:Fatal exception: Empty dictionary! Please check payload or filter. + ... + +In fact, in this situation, FUZZ refers to the previous result (if any):: + + $ wfuzz -z wfuzzp --zD /tmp/test1 -u FUZZ --oF /tmp/test2 + ... + 000000001: 404 11 L 72 W 1558 Ch "http://www.google.com/0" + ... + + $ wfpayload -z wfuzzp --zD /tmp/test2 --efield r.headers.response.date --efield FUZZ[r.headers.response.date] + ... + 000000001: 404 11 L 72 W 1558 Ch "http://www.google.com/0 | Mon, 02 Nov 2020 19:29:03 GMT | Mon, 02 Nov 2020 19:27:27 GMT" + ... + Re-writing a payload """"""" diff --git a/src/wfuzz/dictionaries.py b/src/wfuzz/dictionaries.py index 4ab0d13a..b2ebe413 100644 --- a/src/wfuzz/dictionaries.py +++ b/src/wfuzz/dictionaries.py @@ -1,6 +1,6 @@ from .exception import FuzzExceptNoPluginError, FuzzExceptBadOptions from .facade import Facade -from .filters.ppfilter import FuzzResFilterSlice +from .filters.ppfilter import FuzzResFilterSlice, FuzzResFilter from .fuzzobjects import FuzzWord, FuzzWordType @@ -119,7 +119,8 @@ def next_word(self): class SliceIt(BaseDictionary): def __init__(self, payload, slicestr): - self.ffilter = FuzzResFilterSlice(filter_string=slicestr) + self.ffilter = FuzzResFilter(filter_string=slicestr) + self.ffilter_slice = FuzzResFilterSlice(filter_string=slicestr) self.payload = payload def count(self): @@ -128,10 +129,18 @@ def count(self): def get_type(self): return self.payload.get_type() + def _get_filtered_value(self, item): + if item.type == FuzzWordType.FUZZRES: + filter_ret = self.ffilter.is_visible(item.content) + else: + filter_ret = self.ffilter_slice.is_visible(item.content) + + return filter_ret + def next_word(self): # can be refactored using the walrus operator in python 3.8 item = next(self.payload) - filter_ret = self.ffilter.is_visible(item.content) + filter_ret = self._get_filtered_value(item) if not isinstance(filter_ret, bool) and item.type == FuzzWordType.FUZZRES: raise FuzzExceptBadOptions( @@ -140,7 +149,7 @@ def next_word(self): while isinstance(filter_ret, bool) and not filter_ret: item = next(self.payload) - filter_ret = self.ffilter.is_visible(item.content) + filter_ret = self._get_filtered_value(item) if not isinstance(filter_ret, bool): return FuzzWord(filter_ret, item.type) diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index c453297b..50a0fff3 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -178,7 +178,7 @@ ( "test_desc_assign_fuzz_symbol_op", "-z range,1-1 {}/FUZZ".format(HTTPBIN_URL), - "-z wfuzzp,$$PREVFILE$$ --slice FUZZ[r.url]:=FUZZ[r.url]|replace('1','2') FUZZ[url]", + "-z wfuzzp,$$PREVFILE$$ --slice r.url:=r.url|replace('1','2') FUZZ[url]", ["http://localhost:9000/2"], None, ), From 546bc1129abd14d972e604df7fe9b9484e3a26b1 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 20:50:47 +0100 Subject: [PATCH 77/89] remove wfuzz-cli ref --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d273db33..26bc9833 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -609,7 +609,7 @@ The payload to filter, specified by the -z switch must precede --slice command l The specified expression must return a boolean value, an example, using the unique operator is shown below:: - $ wfuzz-cli.py -z list --zD one-two-one-one --slice "FUZZ|u()" http://localhost:9000/FUZZ + $ wfuzz -z list --zD one-two-one-one --slice "FUZZ|u()" http://localhost:9000/FUZZ ******************************************************** * Wfuzz 2.2 - The Web Fuzzer * From 48d99f41ab83428cdc382ed4798adce9993ec1d4 Mon Sep 17 00:00:00 2001 From: javi Date: Mon, 2 Nov 2020 21:21:40 +0100 Subject: [PATCH 78/89] add test for previous payload filter --- tests/acceptance/test_saved_filter.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/acceptance/test_saved_filter.py diff --git a/tests/acceptance/test_saved_filter.py b/tests/acceptance/test_saved_filter.py new file mode 100644 index 00000000..7125e120 --- /dev/null +++ b/tests/acceptance/test_saved_filter.py @@ -0,0 +1,26 @@ +import pytest +import os +import tempfile + +import wfuzz + + +def get_temp_file(): + temp_name = next(tempfile._get_candidate_names()) + defult_tmp_dir = tempfile._get_default_tempdir() + + return os.path.join(defult_tmp_dir, temp_name) + + +def test_filter_prev_payload(): + + filename = get_temp_file() + for res in wfuzz.get_session("-z range --zD 0-0 -H test:1 -u http://localhost:9000/anything/FUZZ").fuzz(save=filename): + pass + + filename_new = get_temp_file() + for res in wfuzz.get_session("-z wfuzzp --zD {} -u FUZZ -H test:2 --oF {}".format(filename, filename_new)).fuzz(save=filename_new): + pass + + assert len(list(wfuzz.get_session("-z wfuzzp --zD {} --slice r.headers.request.test=2 --dry-run -u FUZZ".format(filename_new)).fuzz())) == 1 + assert len(list(wfuzz.get_session("-z wfuzzp --zD {} --slice FUZZ[r.headers.request.test]=1 --dry-run -u FUZZ".format(filename_new)).fuzz())) == 1 From a3458bff101ec37ffd1055345c7fb75411511a4e Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 3 Nov 2020 00:06:57 +0100 Subject: [PATCH 79/89] add operators tests --- tests/filters/test_filter.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/filters/test_filter.py b/tests/filters/test_filter.py index 905ed5ad..3012b53d 100644 --- a/tests/filters/test_filter.py +++ b/tests/filters/test_filter.py @@ -46,3 +46,22 @@ def test_filter_ret_values_no_response( filter_obj.is_visible(example_full_fuzzres_no_response, filter_string) == expected_result ) + + +@pytest.mark.parametrize( + "filter_string, expected_result", + [ + ("r.cookies.response.nAMe|upper()", "NICHOLAS"), + ("r.cookies.response.name|upper()", "NICHOLAS"), + ("r.cookies.response.name|lower()", "nicholas"), + ("r.cookies.response.name|startswith('N')", True), + ("r.cookies.response.name|replace('N','n')", "nicholas"), + ("'%2e%2e'|unquote()", ".."), + ("'%2e%2f'|decode('urlencode')", "./"), + ("'%%'|encode('urlencode')", "%25%25"), + ], +) +def test_filter_operators( + filter_obj, example_full_fuzzres, filter_string, expected_result +): + assert filter_obj.is_visible(example_full_fuzzres, filter_string) == expected_result From 5dafaa5eb9e6c0ef391a7710961faf6dd3b52125 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 3 Nov 2020 00:15:32 +0100 Subject: [PATCH 80/89] diff operator test --- tests/filters/test_filter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/filters/test_filter.py b/tests/filters/test_filter.py index 3012b53d..505bad5c 100644 --- a/tests/filters/test_filter.py +++ b/tests/filters/test_filter.py @@ -51,6 +51,10 @@ def test_filter_ret_values_no_response( @pytest.mark.parametrize( "filter_string, expected_result", [ + ( + "r.cookies.response.name|diff('test')", + "--- prev\n\n+++ current\n\n@@ -1 +1 @@\n\n-test\n+Nicholas", + ), ("r.cookies.response.nAMe|upper()", "NICHOLAS"), ("r.cookies.response.name|upper()", "NICHOLAS"), ("r.cookies.response.name|lower()", "nicholas"), From 768c04878af9b4a52975f965000c3670ae36c458 Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 3 Nov 2020 00:17:08 +0100 Subject: [PATCH 81/89] formatting --- tests/acceptance/test_saved_filter.py | 34 +++++++++++++++++++--- tests/plugins/test_burplog.py | 41 +++++++++++++-------------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/tests/acceptance/test_saved_filter.py b/tests/acceptance/test_saved_filter.py index 7125e120..1866c74c 100644 --- a/tests/acceptance/test_saved_filter.py +++ b/tests/acceptance/test_saved_filter.py @@ -15,12 +15,38 @@ def get_temp_file(): def test_filter_prev_payload(): filename = get_temp_file() - for res in wfuzz.get_session("-z range --zD 0-0 -H test:1 -u http://localhost:9000/anything/FUZZ").fuzz(save=filename): + for res in wfuzz.get_session( + "-z range --zD 0-0 -H test:1 -u http://localhost:9000/anything/FUZZ" + ).fuzz(save=filename): pass filename_new = get_temp_file() - for res in wfuzz.get_session("-z wfuzzp --zD {} -u FUZZ -H test:2 --oF {}".format(filename, filename_new)).fuzz(save=filename_new): + for res in wfuzz.get_session( + "-z wfuzzp --zD {} -u FUZZ -H test:2 --oF {}".format(filename, filename_new) + ).fuzz(save=filename_new): pass - assert len(list(wfuzz.get_session("-z wfuzzp --zD {} --slice r.headers.request.test=2 --dry-run -u FUZZ".format(filename_new)).fuzz())) == 1 - assert len(list(wfuzz.get_session("-z wfuzzp --zD {} --slice FUZZ[r.headers.request.test]=1 --dry-run -u FUZZ".format(filename_new)).fuzz())) == 1 + assert ( + len( + list( + wfuzz.get_session( + "-z wfuzzp --zD {} --slice r.headers.request.test=2 --dry-run -u FUZZ".format( + filename_new + ) + ).fuzz() + ) + ) + == 1 + ) + assert ( + len( + list( + wfuzz.get_session( + "-z wfuzzp --zD {} --slice FUZZ[r.headers.request.test]=1 --dry-run -u FUZZ".format( + filename_new + ) + ).fuzz() + ) + ) + == 1 + ) diff --git a/tests/plugins/test_burplog.py b/tests/plugins/test_burplog.py index 162005e8..a6320068 100644 --- a/tests/plugins/test_burplog.py +++ b/tests/plugins/test_burplog.py @@ -81,7 +81,7 @@ def readline(self, *args, **kwargs): "HTTP/1.1 200 OK\n" "Server: nginx/1.17.9\n" "\n" - "\n" + '\n' "\n" " \n" "======================================================\n" @@ -104,7 +104,7 @@ def readline(self, *args, **kwargs): "HTTP/1.1 200 OK\n" "Server: nginx/1.17.9\n" "\n" - "\n" + '\n' "\n" "\n" "\n" @@ -128,7 +128,7 @@ def readline(self, *args, **kwargs): "HTTP/1.1 200 OK\n" "Server: nginx/1.17.9\n" "\n" - "\n" + '\n' "\n" "\n" "======================================================\n" @@ -153,7 +153,7 @@ def readline(self, *args, **kwargs): "Date: Mon, 19 Jan 1970 15:36:40 GMT\n" "Last-Modified: Wed, 11 May 2011 10:27:48 GMT\n" "Connection: close\n" - "ETag: \"4dca64a4-156a\"\n" + 'ETag: "4dca64a4-156a"\n' "\n" "\n" "======================================================\n" @@ -161,7 +161,7 @@ def readline(self, *args, **kwargs): "\n" "\n" ), - '', + "", ), ], indirect=["burplog_file"], @@ -225,7 +225,7 @@ def test_burplog_content(burplog_file, expected_content): "Alt-Svc: clear\n" "Connection: close\n" "\n" - "\n" + '\n' "\n" "\n" "======================================================\n" @@ -244,21 +244,20 @@ def test_burplog_content(burplog_file, expected_content): "Connection": "close", }, { - 'Server': 'nginx/1.17.9', - 'Date': 'Sun, 01 Nov 2020 21:35:08 GMT', - 'Content-Type': 'text/xml; charset=utf-8', - 'Content-Length': '42', - 'Strict-Transport-Security': 'max-age=31536000;', - 'X-Content-Type-Options': 'nosniff', - 'Content-Security-Policy': "default-src 'none'; frame-ancestors 'none'", - 'X-Proxy-Cache-Status': 'EXPIRED', - 'Via': '1.1 google', - 'Age': '47', - 'Cache-Control': 'public, max-age=90', - 'Alt-Svc': 'clear', - 'Connection': 'close', - } - + "Server": "nginx/1.17.9", + "Date": "Sun, 01 Nov 2020 21:35:08 GMT", + "Content-Type": "text/xml; charset=utf-8", + "Content-Length": "42", + "Strict-Transport-Security": "max-age=31536000;", + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", + "X-Proxy-Cache-Status": "EXPIRED", + "Via": "1.1 google", + "Age": "47", + "Cache-Control": "public, max-age=90", + "Alt-Svc": "clear", + "Connection": "close", + }, ), ], indirect=["burplog_file"], From b1a7ccb7126b5eb50d3d729ced8e065a9bb7414b Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 3 Nov 2020 00:18:59 +0100 Subject: [PATCH 82/89] diff operator --- docs/user/advanced.rst | 1 + src/wfuzz/filters/ppfilter.py | 22 +++++++++++++++++----- src/wfuzz/helpers/utils.py | 13 +++++++++++++ src/wfuzz/ui/console/mvc.py | 19 +++---------------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 26bc9833..3ab9337e 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -466,6 +466,7 @@ value|replace('what', 'with') value|r('what', 'with') Returns value replacing value|unique() value|u() Returns True if a value is unique. value|startswith('value') value|sw('value') Returns true if the value string starts with param value|gregex('expression') value|gre('exp') Returns first regex group that matches in value +value|diff(expression) Returns diff comparison between value and expression ================================ ======================= ============================================= * When a FuzzResult is available, you could perform runtime introspection of the objects using the following symbols diff --git a/src/wfuzz/filters/ppfilter.py b/src/wfuzz/filters/ppfilter.py index 13032540..1c9ead09 100644 --- a/src/wfuzz/filters/ppfilter.py +++ b/src/wfuzz/filters/ppfilter.py @@ -5,6 +5,7 @@ ) from ..helpers.str_func import value_in_any_list_item from ..helpers.obj_dic import DotDict +from ..helpers.utils import diff import re import collections @@ -64,9 +65,17 @@ def __init__(self, filter_string=None): r"BBB(?:\[(?P(\w|_|-|\.)+)\])?", asMatch=True ).setParseAction(self.__compute_bbb_symbol) + diff_call = Group( + Suppress(Literal("|")) + + Literal("diff") + + Suppress(Literal("(")) + + (fuzz_symbol | res_symbol | bbb_symbol | int_values | quoted_str_value) + + Suppress(")") + ) + fuzz_statement = Group( (fuzz_symbol | res_symbol | bbb_symbol | int_values | quoted_str_value) - + Optional(operator_call, None) + + Optional(diff_call | operator_call, None) ).setParseAction(self.__compute_res_value) operator = oneOf("and or") @@ -127,10 +136,13 @@ def __compute_res_value(self, tokens): if token_tuple: location, operator_match = token_tuple - if operator_match and operator_match.groupdict()["operator"]: - fuzz_val = self._get_operator_value( - location, fuzz_val, operator_match.groupdict() - ) + if location == "diff": + return diff(operator_match, fuzz_val) + else: + if operator_match and operator_match.groupdict()["operator"]: + fuzz_val = self._get_operator_value( + location, fuzz_val, operator_match.groupdict() + ) if isinstance(fuzz_val, list): return [fuzz_val] diff --git a/src/wfuzz/helpers/utils.py b/src/wfuzz/helpers/utils.py index eb6151bb..92576c58 100644 --- a/src/wfuzz/helpers/utils.py +++ b/src/wfuzz/helpers/utils.py @@ -1,4 +1,5 @@ from threading import Lock +import difflib class MyCounter: @@ -20,3 +21,15 @@ def _operation(self, dec): def __call__(self): with self._mutex: return self._count + + +def diff(param1, param2): + delta = difflib.unified_diff( + str(param1).splitlines(False), + str(param2).splitlines(False), + fromfile="prev", + tofile="current", + n=0, + ) + + return "\n".join(delta) diff --git a/src/wfuzz/ui/console/mvc.py b/src/wfuzz/ui/console/mvc.py index 5aa0d70f..37e7bb8e 100644 --- a/src/wfuzz/ui/console/mvc.py +++ b/src/wfuzz/ui/console/mvc.py @@ -1,7 +1,6 @@ import sys from collections import defaultdict import threading -import difflib try: from itertools import zip_longest @@ -162,7 +161,7 @@ def __init__(self, session_options): self.term = Term() self.printed_lines = 0 - def _print_verbose(self, res, print_nres=True, extra_description=None): + def _print_verbose(self, res, print_nres=True): txt_colour = ( Term.noColour if not res.is_baseline or not self.colour else Term.fgCyan ) @@ -179,10 +178,6 @@ def _print_verbose(self, res, print_nres=True, extra_description=None): if "Server" in res.history.headers.response: server = res.history.headers.response["Server"] - description = res.description - if extra_description: - description = "{} | diff:\n{}".format(res.description, extra_description) - rows = [ ("%09d:" % res.nres if print_nres else " |_", txt_colour), ("%.3fs" % res.timer, txt_colour), @@ -195,7 +190,7 @@ def _print_verbose(self, res, print_nres=True, extra_description=None): ("%d Ch" % res.chars, txt_colour), (server, txt_colour), (location, txt_colour), - ('"%s"' % description, txt_colour), + ('"%s"' % res.description, txt_colour), ] self.term.set_colour(txt_colour) @@ -308,15 +303,7 @@ def result(self, res): ): prev_res = res.payload_man.get_payload_content(1) if self.verbose: - delta = difflib.unified_diff( - prev_res.history.raw_content.splitlines(True), - res.history.raw_content.splitlines(True), - fromfile="prev", - tofile="current", - ) - self._print_verbose( - prev_res, print_nres=False, extra_description="".join(delta) - ) + self._print_verbose(prev_res, print_nres=False) else: self._print(prev_res, print_nres=False) From e3876722745f3887c6966bc422ce121b32b001bf Mon Sep 17 00:00:00 2001 From: javi Date: Tue, 3 Nov 2020 13:38:44 +0100 Subject: [PATCH 83/89] fix py34 tests --- src/wfuzz/plugin_api/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/plugin_api/base.py b/src/wfuzz/plugin_api/base.py index 38457be6..14b836f1 100644 --- a/src/wfuzz/plugin_api/base.py +++ b/src/wfuzz/plugin_api/base.py @@ -24,7 +24,7 @@ def __init__(self): # check mandatory params, assign default values for name, default_value, required, description in self.parameters: - param_name = f"{self.name}.{name}" + param_name = "{}.{}".format(self.name, name) if required and param_name not in list(self.kbase.keys()): raise FuzzExceptBadOptions( From 43bd5f0be0f89ab3c9c39a89cb1fcfabaf39628b Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 00:12:21 +0100 Subject: [PATCH 84/89] text parses does not update index when reading full string --- src/wfuzz/externals/reqresp/TextParser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/externals/reqresp/TextParser.py b/src/wfuzz/externals/reqresp/TextParser.py index a8553db8..28e97a94 100755 --- a/src/wfuzz/externals/reqresp/TextParser.py +++ b/src/wfuzz/externals/reqresp/TextParser.py @@ -130,9 +130,8 @@ def readLine(self): if self.oldindex >= 0: self.newindex = self.string.find("\n", self.oldindex, len(self.string)) if self.newindex == -1: - self.lastFull_line = self.string[self.oldindex : len(self.string)] - else: - self.lastFull_line = self.string[self.oldindex : self.newindex + 1] + self.newindex = len(self.string) - 1 + self.lastFull_line = self.string[self.oldindex : self.newindex + 1] self.oldindex = self.newindex + 1 else: From d02966a9cba95f1d5456a6c5d47a7e983b5ab4df Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 00:16:37 +0100 Subject: [PATCH 85/89] more restrictive http protocol regex --- src/wfuzz/externals/reqresp/Response.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wfuzz/externals/reqresp/Response.py b/src/wfuzz/externals/reqresp/Response.py index 52a7bd97..c456cd19 100644 --- a/src/wfuzz/externals/reqresp/Response.py +++ b/src/wfuzz/externals/reqresp/Response.py @@ -146,7 +146,7 @@ def parseResponse(self, rawheader, rawbody=None, type="curl"): tp = TextParser() tp.setSource("string", rawheader) - tp.readUntil(r"(HTTP\S*) ([0-9]+)") + tp.readUntil(r"(HTTP/[0-9.]+) ([0-9]+)") while True: while True: try: @@ -162,7 +162,7 @@ def parseResponse(self, rawheader, rawbody=None, type="curl"): if self.code != "100": break else: - tp.readUntil(r"(HTTP\S*) ([0-9]+)") + tp.readUntil(r"(HTTP/[0-9.]+) ([0-9]+)") self.code = int(self.code) @@ -176,7 +176,7 @@ def parseResponse(self, rawheader, rawbody=None, type="curl"): # curl sometimes sends two headers when using follow, 302 and the final header # also when using proxies tp.readLine() - if not tp.search(r"(HTTP\S*) ([0-9]+)"): + if not tp.search(r"(HTTP/[0-9.]+) ([0-9]+)"): break else: self._headers = [] From f6ae03858228ea314003120d2eb41c70bf5bf98f Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 00:21:10 +0100 Subject: [PATCH 86/89] remove future lib --- requirements.txt | 1 - setup.py | 1 - src/wfuzz/facade.py | 5 +---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 44d01ad6..daf8a36f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ # attrs==20.1.0 # via pytest chardet==3.0.4 # via wfuzz (setup.py) -future==0.18.2 # via wfuzz (setup.py) iniconfig==1.0.1 # via pytest more-itertools==8.5.0 # via pytest packaging==20.4 # via pytest diff --git a/setup.py b/setup.py index 4ef20099..d75874ec 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ 'pycurl', 'pyparsing<2.4.2;python_version<="3.4"', 'pyparsing>=2.4*;python_version>="3.5"', - 'future', 'six', 'configparser;python_version<"3.5"', 'chardet', diff --git a/src/wfuzz/facade.py b/src/wfuzz/facade.py index 4f4f4462..e427377c 100644 --- a/src/wfuzz/facade.py +++ b/src/wfuzz/facade.py @@ -8,8 +8,6 @@ import os -# python2 and 3: metaclass -from future.utils import with_metaclass ERROR_CODE = -1 BASELINE_CODE = -2 @@ -64,8 +62,7 @@ def get_plugin(self, identifier): ) -# python2 and 3: class Facade(metaclass=utils.Singleton): -class Facade(with_metaclass(Singleton, object)): +class Facade(metaclass=Singleton): def __init__(self): self.__plugins = dict( From 4aec2b165aeaff4c4d7d3167bd0f2af17383a7cf Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 00:21:30 +0100 Subject: [PATCH 87/89] add burplog test --- tests/plugins/test_burplog.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/plugins/test_burplog.py b/tests/plugins/test_burplog.py index a6320068..cef0c6ab 100644 --- a/tests/plugins/test_burplog.py +++ b/tests/plugins/test_burplog.py @@ -138,6 +138,26 @@ def readline(self, *args, **kwargs): ), '\n\n', ), + ( + ( + "======================================================\n" + "2:17:05 PM https://www.xxx.es:443 [2.2.2.1]\n" + "======================================================\n" + "GET /sttc/dbook-fp/ctrip-prod-2.4.0.min.js HTTP/1.1\n" + "Host: www.xxx.es\n" + "\n" + "\n" + "======================================================\n" + "HTTP/1.1 200 OK\n" + "\n" + "HTTP\",\" 333D Visionplugin\n" + "======================================================\n" + "\n" + "\n" + "\n" + ), + "HTTP\",\" 333D Visionplugin", + ), ( ( "======================================================\n" From c765e870c18bfb067508cb6375a8f6d642a1d6d8 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 00:52:47 +0100 Subject: [PATCH 88/89] bump version --- src/wfuzz/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wfuzz/__init__.py b/src/wfuzz/__init__.py index 02950616..159dab7e 100644 --- a/src/wfuzz/__init__.py +++ b/src/wfuzz/__init__.py @@ -1,5 +1,5 @@ __title__ = "wfuzz" -__version__ = "3.0.3" +__version__ = "3.1.0" __build__ = 0x023000 __author__ = "Xavier Mendez" __license__ = "GPL 2.0" From 3198d6066a553e2d517757207dada9c196489f47 Mon Sep 17 00:00:00 2001 From: Xavi Mendez Date: Fri, 6 Nov 2020 09:57:11 +0100 Subject: [PATCH 89/89] black format --- tests/plugins/test_burplog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/plugins/test_burplog.py b/tests/plugins/test_burplog.py index cef0c6ab..1f05cb86 100644 --- a/tests/plugins/test_burplog.py +++ b/tests/plugins/test_burplog.py @@ -150,13 +150,13 @@ def readline(self, *args, **kwargs): "======================================================\n" "HTTP/1.1 200 OK\n" "\n" - "HTTP\",\" 333D Visionplugin\n" + 'HTTP"," 333D Visionplugin\n' "======================================================\n" "\n" "\n" "\n" ), - "HTTP\",\" 333D Visionplugin", + 'HTTP"," 333D Visionplugin', ), ( (