Merge pull request xmendez#102 from xmendez/hotfixes_2.3.1

Hotfixes 2.3.1

setup.py

@@ -22,7 +22,8 @@
     'pyparsing>=2.2.0',
     'future>=0.16.0',
     'six>=1.10.0',
-    'configparser>=3.5.0'
+    'configparser>=3.5.0',
+    'chardet',
 ]
 
 setup(

src/wfuzz/__init__.py

@@ -1,5 +1,5 @@
 __title__ = 'wfuzz'
-__version__ = "2.3"
+__version__ = "2.3.1"
 __build__ = 0x023000
 __author__ = 'Xavier Mendez'
 __license__ = 'GPL 2.0'

src/wfuzz/plugins/iterators/iterations.py

@@ -42,8 +42,8 @@ class product(object):
     priority = 99
 
     def __init__(self, *i):
-        self.it = itertools.product(*i)
         self.__count = reduce(lambda x, y: x * y.count(), i[1:], i[0].count())
+        self.it = itertools.product(*i)
 
     def count(self):
         return self.__count

src/wfuzz/plugins/payloads/file.py

@@ -1,6 +1,7 @@
 from wfuzz.externals.moduleman.plugin import moduleman_plugin
 from wfuzz.exception import FuzzExceptBadFile
 from wfuzz.plugin_api.base import BasePayload
+from wfuzz.utils import open_file_detect_encoding
 
 
 @moduleman_plugin
@@ -25,18 +26,18 @@ def __init__(self, params):
         BasePayload.__init__(self, params)
 
         try:
-            self.f = open(self.find_file(self.params["fn"]), "r")
+            self.f = open_file_detect_encoding(self.find_file(self.params["fn"]))
         except IOError as e:
             raise FuzzExceptBadFile("Error opening file. %s" % str(e))
 
         self.__count = None
 
     def __next__(self):
-        line = self.f.readline().strip()
-        if line == '':
+        line = self.f.readline()
+        if not line:
             self.f.close()
             raise StopIteration
-        return line
+        return line.strip()
 
     def count(self):
         if self.__count is None:

src/wfuzz/utils.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import six
+from chardet.universaldetector import UniversalDetector
 
 
 def json_minify(string, strip_space=True):
@@ -130,3 +131,23 @@ def convert_to_unicode(text):
         return text.encode("utf-8", errors='ignore')
     else:
         return text
+
+
+def open_file_detect_encoding(file_path):
+    def detect_encoding(file_path):
+        detector = UniversalDetector()
+        detector.reset()
+
+        with open(file_path, mode='rb') as file_to_detect:
+            for line in file_to_detect:
+                detector.feed(line)
+                if detector.done:
+                    break
+        detector.close()
+
+        return detector.result
+
+    if sys.version_info >= (3, 0):
+        return open(file_path, "r", encoding=detect_encoding(file_path).get('encoding', 'utf-8'))
+    else:
+        return open(file_path, "r")