acceptance tests: docker-compose for http and proxy

tests/server_dir/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.6-alpine
+ADD . /code
+WORKDIR /code
+EXPOSE 8000/tcp
+CMD ["python", "simple_server.py"]

tests/server_dir/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '2'
+services:
+  httpserver:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "8000:8000"
+    command: python simple_server.py
+  httpproxy:
+    image: mitmproxy/mitmproxy
+    ports:
+      - "8080:8080"
+    command: mitmdump

tests/simple_server.py → tests/server_dir/simple_server.py

@@ -1,14 +1,14 @@
 # slightly modified from
 # https://gist.github.com/trungly/5889154
 
-from BaseHTTPServer import HTTPServer
-import urlparse
-from SimpleHTTPServer import SimpleHTTPRequestHandler
+import urllib.parse
+from http.server import HTTPServer
+from http.server import SimpleHTTPRequestHandler
 
 
 class GetHandler(SimpleHTTPRequestHandler):
     def do_HEAD(self):
-        parsed_path = urlparse.urlparse(self.path)
+        parsed_path = urllib.parse.urlparse(self.path)
         if parsed_path.path.startswith("/echo"):
             message = '\n'.join([
                 'CLIENT VALUES:',
@@ -24,7 +24,7 @@ def do_HEAD(self):
                 ])
             self.send_response(200)
             self.end_headers()
-            self.wfile.write(message)
+            self.wfile.write(message.encode('utf-8'))
         elif parsed_path.path.startswith("/redirect"):
             self.send_response(301)
             self.send_header('Location', "/echo")
@@ -35,7 +35,7 @@ def do_HEAD(self):
         return
 
     def do_GET(self):
-        parsed_path = urlparse.urlparse(self.path)
+        parsed_path = urllib.parse.urlparse(self.path)
         if parsed_path.path.startswith("/echo"):
             message = '\n'.join([
                 'CLIENT VALUES:',
@@ -51,7 +51,7 @@ def do_GET(self):
                 ])
             self.send_response(200)
             self.end_headers()
-            self.wfile.write(message)
+            self.wfile.write(message.encode('utf-8'))
         elif parsed_path.path.startswith("/redirect"):
             self.send_response(301)
             self.send_header('Location', "/echo")
@@ -62,10 +62,10 @@ def do_GET(self):
         return
 
     def do_POST(self):
-        parsed_path = urlparse.urlparse(self.path)
+        parsed_path = urllib.parse.urlparse(self.path)
         if parsed_path.path.startswith("/echo"):
-            content_len = int(self.headers.getheader('content-length'))
-            post_body = self.rfile.read(content_len)
+            content_len = int(self.headers.get('content-length'))
+            post_body = self.rfile.read(content_len).decode('utf-8')
             self.send_response(200)
             self.end_headers()
 
@@ -84,12 +84,11 @@ def do_POST(self):
                 '',
                 ])
 
-            self.wfile.write(message)
+            self.wfile.write(message.encode('utf-8'))
 
         return
 
 
 if __name__ == '__main__':
-    server = HTTPServer(('localhost', 8080), GetHandler)
-    print 'Starting server at http://localhost:8080'
+    server = HTTPServer(('0.0.0.0', 8000), GetHandler)
     server.serve_forever()

tests/acceptance.py → tests/test_acceptance.py

@@ -1,14 +1,7 @@
-import sys
 import os
 import unittest
-import multiprocessing
 import tempfile
-from miproxy.proxy import AsyncMitmProxy
 
-from simple_server import GetHandler
-from BaseHTTPServer import HTTPServer
-
-sys.path.insert(0, os.path.abspath('../src'))
 import wfuzz
 
 
@@ -18,6 +11,10 @@
 
 ECHO_URL = "%s:8000/echo" % (LOCAL_DOMAIN)
 
+# docker containers with HTTP server and proxy must be started before running these tests
+# cd tests/server_dir
+# docke-compose up
+
 # IDEAS:
 #
 # baseline duplicated with single filters
@@ -44,7 +41,7 @@
     # fuzzing HTTP values
     ("test_basic_path_fuzz", "%s/FUZZ" % URL_LOCAL, [["a", "b", "c"]], dict(), [(200, '/dir/a'), (200, '/dir/b'), (200, '/dir/c')], None),
     ("test_multi_path_fuzz", "%s/FUZZ/FUZ2Z/FUZ3Z" % ECHO_URL, [["a"], ["b"], ["c"]], dict(filter="content~'path=/echo/a/b/c'"), [(200, '/echo/a/b/c')], None),
-    ("test_basic_method_fuzz", "%s" % URL_LOCAL, [["OPTIONS", "PUT"]], dict(method="FUZZ", filter="content~'Unsupported method' and content~FUZZ"), [(501, '/dir'), (501, '/dir')], None),
+    ("test_basic_method_fuzz", "%s" % URL_LOCAL, [["OPTIONS", "HEAD"]], dict(method="FUZZ", filter="content~'Unsupported method' and content~FUZZ"), [(501, '/dir')], None),
     ("test_basic_postdata_fuzz", "%s" % ECHO_URL, [["onevalue", "twovalue"]], dict(postdata="a=FUZZ", filter="content~FUZZ and content~'POST_DATA=a='"), [(200, '/echo'), (200, '/echo')], None),
     ("test_basic_postdata2_fuzz", "%s" % ECHO_URL, [["onevalue", "twovalue"]], dict(postdata="FUZZ=1234", filter="content~'POST_DATA=twovalue=1234' or content~'POST_DATA=onevalue=1234'"), [(200, '/echo'), (200, '/echo')], None),
     ("test_basic_postdata3_fuzz", "%s" % ECHO_URL, [["onevalue", "twovalue"]], dict(postdata="FUZZ", filter="content~'POST_DATA=twovalue' or content~'POST_DATA=onevalue'"), [(200, '/echo'), (200, '/echo')], None),
@@ -168,8 +165,16 @@ def test(self):
 
         # repeat test with extra params if specified and check against
         if extra_params:
-            with wfuzz.FuzzSession(url=url) as s:
-                same_list = map(lambda x: (x.code, x.history.urlparse.path), s.get_payloads(payloads).fuzz(**extra_params))
+            # if using proxy change localhost for docker compose service
+            proxied_url = url
+            proxied_payloads = payloads
+            if "proxies" in extra_params:
+                proxied_url = url.replace('localhost', 'httpserver')
+                if payloads:
+                    proxied_payloads = [[payload.replace("localhost", "httpserver") for payload in payloads_list] for payloads_list in payloads]
+
+            with wfuzz.FuzzSession(url=proxied_url) as s:
+                same_list = [(x.code, x.history.urlparse.path) for x in s.get_payloads(proxied_payloads).fuzz(**extra_params)]
 
             self.assertEqual(sorted(ret_list), sorted(same_list))
         else:
@@ -214,7 +219,6 @@ def test(self):
 
         # repeat test with performaing FUZZ[url] saved request
         with wfuzz.FuzzSession(payloads=[("wfuzzp", dict(fn=filename))], url="FUZZ[url]") as s:
-            print filename
             same_list = map(lambda x: (x.code, x.history.urlparse.path), s.fuzz())
 
         self.assertEqual(sorted(ret_list), sorted(same_list))
@@ -321,37 +325,7 @@ def create_tests():
         duplicate_tests_diff_params(basic_tests, "_proxy_", dict(proxies=[("localhost", 8080, "HTML")]), None)
 
 
-if __name__ == '__main__':
+create_tests()
 
-    httpd = None
-    proxyd = None
-    httpd_server_process = None
-    server_process = None
-
-    try:
-        # Setup simple HTTP sever
-        os.chdir("server_dir")
-        httpd = HTTPServer(('localhost', HTTPD_PORT), GetHandler)
-
-        httpd_server_process = multiprocessing.Process(target=httpd.serve_forever)
-        httpd_server_process.daemon = True
-        httpd_server_process.start()
-
-        # HTTP proxy
-        proxyd = AsyncMitmProxy()
-
-        server_process = multiprocessing.Process(target=proxyd.serve_forever)
-        server_process.daemon = True
-        server_process.start()
-
-        create_tests()
-        unittest.main()
-    finally:
-        if httpd:
-            httpd.server_close()
-        if proxyd:
-            proxyd.server_close()
-        if server_process:
-            server_process.terminate()
-        if httpd_server_process:
-            httpd_server_process.terminate()
+if __name__ == '__main__':
+    unittest.main()

tests/test_api.py

@@ -68,7 +68,7 @@ def test_payload(self):
         server_dir = os.path.join(wfuzz.utils.get_path('../../tests'), 'server_dir')
 
         payload_list = list(wfuzz.payload(**{'payloads': [('dirwalk', {'default': server_dir, 'encoder': None}, None)]}))
-        self.assertEqual(payload_list, [('ca.pem',), ('iterators/bb',), ('iterators/ac',), ('iterators/aa',), ('static/placeholder.txt',), ('plugins/robots.txt',), ('recursive_dir/a/b/c/placeholder.txt',), ('dir/b',), ('dir/c',), ('dir/one',), ('dir/a',)])
+        self.assertEqual(payload_list, [('Dockerfile',), ('docker-compose.yml',), ('simple_server.py',), ('iterators/bb',), ('iterators/ac',), ('iterators/aa',), ('static/placeholder.txt',), ('plugins/robots.txt',), ('recursive_dir/a/b/c/placeholder.txt',), ('dir/b',), ('dir/c',), ('dir/one',), ('dir/a',)])
 
         payload_list = list(wfuzz.payload(**{'payloads': [('file', {'default': os.path.join(os.path.join(server_dir, 'dir'), 'b'), 'encoder': None}, None)]}))
         self.assertEqual(payload_list, [('one',), ('two',)])