Merge pull request xmendez#213 from xmendez/dev

Merge Dev to master
edley · Aug 19, 2020 · f3b407f · f3b407f
2 parents 1bba4c3 + 68ab0e0
commit f3b407f
Show file tree

Hide file tree

Showing 133 changed files with 8,083 additions and 3,837 deletions.
diff --git a/.flake8 b/.flake8
@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 80
+select = C,E,F,W,B,B950
+ignore = E203, E501, W503, E402, F401, W504
diff --git a/.travis.yml b/.travis.yml
@@ -15,9 +15,10 @@ install:
  - python setup.py install
 
 script:
-  - flake8 --ignore=E501,E402,F401,W504 src tests
+  - flake8 src tests
   - coverage run --append -m unittest discover -v -s tests/
   - if [[ $TRAVIS_PYTHON_VERSION == '3.6' && $TRAVIS_BRANCH == 'master' ]]; then codecov; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != '3.4' && $TRAVIS_PYTHON_VERSION != '3.5' ]]; then black --check src tests; fi
 deploy:
   provider: pypi
   user: x4vi_mendez

diff --git a/Makefile b/Makefile
@@ -3,8 +3,8 @@ test:
 	pip install tox
 	tox --recreate
 flake8:
-	pip install flake8
-	flake8 --ignore=E501,E402,F401,W504 src tests
+	black --check src tests
+	flake8 src tests
 publish:
 	pip install 'twine>=1.5.0'
 	python setup.py sdist
@@ -21,7 +21,7 @@ docs:
 	cd docs && make html
 
 coverage:
-	coverage report --skip-covered --include "*python3.5/site-packages/wfuzz*" -m
+	coverage report --skip-covered --include "*python3.8/site-packages/wfuzz*" -m
 
 install: install-dev
 	pip install -r requirements.txt

diff --git a/docs/conf.py b/docs/conf.py
@@ -93,7 +93,8 @@
     'github_user': 'xmendez',
     'github_repo': 'wfuzz',
     'github_banner': True,
-    'show_related': False
+    'show_related': False,
+    'sidebar_collapse': True,
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,
@@ -109,7 +110,11 @@
 
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
-    '**':       ['sidebarlogo.html', 'globaltoc.html']
+    '**': [
+        'sidebarlogo.html',
+        'navigation.html',
+        'searchbox.html'
+    ]
 }
 
 # -- Options for LaTeX output ---------------------------------------------

diff --git a/docs/index.rst b/docs/index.rst
@@ -18,9 +18,10 @@ Wfuzz: The Web fuzzer
 .. image:: https://codecov.io/github/xmendez/wfuzz/coverage.svg?branch=master
     :target: https://codecov.io/github/xmendez/wfuzz
 
-Wfuzz supports Python 3. The use of **Python 3** is preferred (and faster) over Python 2. 
+Wfuzz provides a framework to automate web applications security assessments and could help you to secure your web applications by finding and exploiting web application vulnerabilities.
 
-See Wfuzz in action:
+See Wfuzz in action
+-------------------
 
 * Wfuzz cli::
 
@@ -78,33 +79,46 @@ other tools included in the wfuzz framework.
     $ wfencode -e md5 test
     098f6bcd4621d373cade4e832627b4f6
 
-Wfuzz has been created to facilitate the task in web applications assessments and it is based on a simple concept: it replaces any reference to the FUZZ keyword by the value of a given payload.
+
+How it works
+------------
+
+Wfuzz it is based on a simple concept: it replaces any reference to the FUZZ keyword by the value of a given payload.
 
 A payload in Wfuzz is a source of data.
 
 This simple concept allows any input to be injected in any field of an HTTP request, allowing to perform complex web security attacks in different web application components such as: parameters, authentication, forms, directories/files, headers, etc.
 
-Wfuzz is more than a web content scanner:
+Wfuzz is more than a web brute forcer:
 
-- Wfuzz could help you to secure your web applications by finding and exploiting web application vulnerabilities. Wfuzz's web application vulnerability scanner is supported by plugins.
+- Wfuzz's web application vulnerability scanner is supported by plugins.
 - Wfuzz is a completely modular framework and makes it easy for even the newest of Python developers to contribute. Building plugins is simple and takes little more than a few minutes.
 - Wfuzz exposes a simple language interface to the previous HTTP requests/responses performed using Wfuzz or other tools, such as Burp. This allows you to perform manual and semi-automatic tests with full context and understanding of your actions, without relying on a web application scanner underlying implementation.
 
-User Guide	
-==================	
+Installation Guide
+==================
 
 .. toctree::	
-   :maxdepth: 2	
+   :maxdepth: 4
+
+   user/installation
+   user/breaking
+
+User Guide
+==================
+
+.. toctree::
+   :maxdepth: 4
 
-   user/installation	
-   user/getting	
-   user/basicusage	
-   user/advanced	
+   user/getting
+   user/basicusage
+   user/advanced
+   user/wfpayload
 
 Library Guide	
 ==================	
 
 .. toctree::	
-   :maxdepth: 2	
+   :maxdepth: 4
 
    library/guide
diff --git a/docs/library/guide.rst b/docs/library/guide.rst
@@ -1,6 +1,11 @@
-Library Options
+Python library
 ===============
 
+Wfuzz's Python library allows to automate tasks and integrate Wfuzz into new tools or scripts.
+
+Library Options
+---------------
+
 All options that are available within the Wfuzz command line interface are available as library options:
 
 ======================== =====================================================================================
@@ -10,7 +15,7 @@ CLI Option               Library Option
 --recipe <filename>      recipe=["filename"]
 --oF <filename>          save="filename"
 -f filename,printer      printer=("filename", "printer")
---dry-run                dryrun=True
+--dry-run                transport="dryrun"
 -p addr                  proxies=[("ip","port","type")]
 -t N                     concurrent=N
 -s N                     delay=0.0
@@ -30,7 +35,7 @@ CLI Option               Library Option
 --sc/sl/sw/sh N[,N]+     sc/sl/sw/sh=[N,N]
 --ss/hs regex            ss/hs="regex"
 --filter <filter>        filter="filter exp"
---prefilter <filter>     prefilter="prefilter exp"
+--prefilter <filter>     prefilter=["prefilter exp"]
 -b cookie                cookie=["cookie1=value1",]
 -d postdata              postdata="postdata"
 -H header                headers=[("header1", "value1"),]
@@ -40,7 +45,7 @@ CLI Option               Library Option
 These options can be used in the main library interfaces: fuzz, payload or session indistinctly.
 
 Fuzzing a URL
-=============
+-------------
 
 Fuzzing a URL with wfuzz library is very simple. Firstly, import the wfuzz module::
 
@@ -61,7 +66,7 @@ Now, let's try to fuzz a web page to look for hidden content, such as directorie
 Now, we have a FuzzResult object called r. We can get all the information we need from this object.
 
 FuzzSession object
-==================
+------------------
 
 A FuzzSession object has all the methods of the main wfuzz API.
 
@@ -87,7 +92,7 @@ FuzzSession can also be used as context manager::
 
 
 Get payload
-===========
+-----------
 
 The get_payload function generates a Wfuzz payload from a Python iterable. It is a quick and flexible way of getting a payload programmatically without using Wfuzz payloads plugins.
 
@@ -123,7 +128,7 @@ The get_payloads method can be used when various payloads are needed::
     00019:  C=404      7 L        12 W          168 Ch        "0 - a"
 
 Get session
-===========
+-----------
 
 The get_session function generates a Wfuzz session object from the specified command line. It is a quick way of getting a payload programmatically from a string representing CLI options::
 
@@ -146,7 +151,7 @@ The get_session function generates a Wfuzz session object from the specified com
     00010:  C=404      7 L        12 W          168 Ch        "9"
 
 Interacting with the results
-============================
+----------------------------
 
 Once a Wfuzz result is available the grammar defined in the filter language can be used to work with the results' values. For example::
 

diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst
@@ -490,7 +490,8 @@ FuzzRequest object's attribute (you need to use the r. prefix) such as:
 ============================ =============================================
 Name                         Description
 ============================ =============================================
-url                          HTTP request's value
+url                          HTTP request's url
+urlp                         HTTP request's parsed url (see section below).
 method                       HTTP request's verb
 scheme                       HTTP request's scheme
 host                         HTTP request's host
@@ -504,8 +505,8 @@ cookies.response.<<name>>    Specified HTTP response cookie
 headers.all                  All HTTP request and response headers
 headers.request              HTTP request headers
 headers.response             HTTP response headers
-headers.request.<<name>>     Specified HTTP request given header
-headers.response.<<name>>    Specified HTTP response given header
+headers.request.<<name>>     Specified HTTP request header case insensitive
+headers.response.<<name>>    Specified HTTP response header insensitive
 params.all                   All HTTP request GET and POST parameters
 params.get                   All HTTP request GET parameters
 params.post                  HTTP request POST parameters in returned as a dictionary
@@ -597,18 +598,18 @@ Results with plugin issues can be filter as well::
 
     $ wfuzz -z list --zD index -u http://testphp.vulnweb.com/FUZZ.php --script headers --filter "plugins~'nginx'"
 
-Filtering a payload
+Payload mangling
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Slice
+Slicing a payload
 """""""
 
-The --slice command line parameter in conjunction with the described filter language allows you to filter a payload.
+The --slice command line parameter in conjunction with the described language allows you to filter a payload.
 The payload to filter, specified by the -z switch must precede --slice command line parameter.
 
-An example is shown below::
+The specified expression must return a boolean value, an example, using the unique operator is shown below::
 
-    $ wfuzz-cli.py -z list,one-two-one-one --slice "FUZZ|u()" http://localhost:9000/FUZZ
+    $ wfuzz-cli.py -z list --zD one-two-one-one --slice "FUZZ|u()" http://localhost:9000/FUZZ
 
     ********************************************************
     * Wfuzz 2.2 - The Web Fuzzer                           *
@@ -629,9 +630,22 @@ An example is shown below::
     Filtered Requests: 0
     Requests/sec.: 62.85908
     
-It is worth noting that the type of payload dictates the available language symbols. For example, a dictionary payload such as the one in the example
+It is worth noting that, the type of payload dictates the available language symbols. For example, a dictionary payload such as in the example
 above does not have a full FuzzResult object context and therefore object fields cannot be used.
 
+Re-writing a payload
+"""""""
+
+The slice command parameter also allows to re-write a payload. Any value, other than a boolean, returned by the
+specified expression will be interpreted not to filter the source payload but to change its value.
+
+For example::
+
+    $ ./wfuzz -z list --zD one-two-three --slice "FUZZ|upper()" -u https://www.wfuzz.io/FUZZ
+    000000001:   404        11 L     72 W     1560 Ch     "ONE"
+    000000003:   404        11 L     72 W     1562 Ch     "THREE"
+    000000002:   404        11 L     72 W     1560 Ch     "TWO"
+
 Prefilter
 """""""""
 
@@ -640,6 +654,8 @@ performed just before any HTTP request is done.
 
 In this context you are filtering a FuzzResult object, which is the result of combining all the input payloads, that is has not been updated with the result of performing its associated HTTP request yet and therefore lacking some information.
 
+The --prefilter command cannot be used to re-write a payload. The assignment operators can be used to modify the FuzzResult object's fields but expressions other booleans will be ignored.
+
 Reutilising previous results
 --------------------------------------
 
@@ -724,7 +740,10 @@ The above command will generate HTTP requests such as the following::
 
 You can filter the payload using the filter grammar as described before.
 
-The assignment operators can be used to modify previous requests easily, for example, let's add a quote to every parameter looking for SQL injection issues::
+Request mangling
+^^^^^^^^^
+
+The assignment operators can be used to modify previous requests, for example, let's add a quote to every string parameter prior of performing the HTTP request::
 
     $ wfuzz -z range,1-5 --oF /tmp/session http://testphp.vulnweb.com/artists.php?artist=FUZZ
     000003:  C=200    118 L      455 W         5326 Ch        "3"
@@ -736,25 +755,5 @@ The assignment operators can be used to modify previous requests easily, for exa
     |_  Error identified: Warning: mysql_fetch_array()
     ...
 
-wfpayload
-^^^^^^^^^
-
-If you do not want to perform any request, just find some specific HTTP request you can use the wfpayload executable.
-
-For example, the following will return a unique list of HTTP requests including the authtoken parameter as a GET parameter::
-
-    $ wfpayload -z burplog,a_burp_log.log --slice "params.get~'authtoken' and url.pstrip|u()"
-
-Authtoken is the parameter used by BEA WebLogic Commerce Servers (TM) as a CSRF token, and therefore the above will find all the requests exposing the CSRF token in the URL.
-
-You can also select the field to show, for example::
-
-    $ wfpayload -z wfuzzp --zD /tmp/session --field r.params.get
-    artist=5
-    ...
-
-Or::
 
-    $ wfpayload -z wfuzzp --zD /tmp/session --efield r.params.get
-    000000006:   200        99 L     272 W    3868 Ch     "5 | artist=5"
-    ...
+The above command looks for simple SQL injection issues.
diff --git a/docs/user/basicusage.rst b/docs/user/basicusage.rst
@@ -252,15 +252,31 @@ For example, to show results in JSON format use the following command::
 
     $ wfuzz -o json -w wordlist/general/common.txt http://testphp.vulnweb.com/FUZZ
 
-When using the default output you can also select an additional FuzzResult's field to show together with the payload description::
+When using the default output you can also select additional FuzzResult's fields to show, using --efield, together with the payload description::
 
-    $ wfuzz -z range --zD 0-1 -u http://testphp.vulnweb.com/artists.php?artist=FUZZ --field r
+    $ wfuzz -z range --zD 0-1 -u http://testphp.vulnweb.com/artists.php?artist=FUZZ --efield r
     ...
     000000001:   200        99 L     272 W    3868 Ch   0 | GET /artists.php?artist=0 HTTP/1.1
                                                         Content-Type: application/x-www-form-urlencoded
                                                         User-Agent: Wfuzz/2.4
                                                         Host: testphp.vulnweb.com
     ...
 
+The above is useful, for example, to debug what exact HTTP request Wfuzz sent to the remote Web server.
 
-The above is useful, for example, to debug what exact HTTP request Wfuzz sent to the remote Web server. Check the filter language section in the advance usage document for the available fields.
+To completely replace the default payload output you can use --field instead::
+
+    $ wfuzz -z range --zD 0-1 -u http://testphp.vulnweb.com/artists.php?artist=FUZZ --field url
+    ...
+    000000001:   200        104 L    364 W    4735 Ch     "http://testphp.vulnweb.com/artists.php?artist=0"
+    ...
+
+--efield and --field can be repeated to show several fields::
+
+
+    $ wfuzz -z range --zD 0-1 -u http://testphp.vulnweb.com/artists.php?artist=FUZZ --efield url --efield h
+    ...
+    000000001:   200        104 L    364 W    4735 Ch     "0 | http://testphp.vulnweb.com/artists.php?artist=0 | 4735"                                                                                        
+    ...
+
+--efield and --field are in fact filter expressions. Check the filter language section in the advance usage document for the available fields and operators.
diff --git a/docs/user/breaking.rst b/docs/user/breaking.rst
@@ -0,0 +1,8 @@
+Breaking changes
+=============
+
+Following https://semver.org/ versioning since Wfuzz 3.0.0.
+
+* Wfuzz 3.0.0:
+    * In wfuzz library prefilter is a list of filters not a string.
+    * When using --recipe, stored options that are a list are appended. Previously, the last one took precedence.