Merge branch 'feature/refactor_encoding' into develop

Author: Marcel Radischat

pandasqt/_lib/magic/db/magic.mgc

@@ -0,0 +1,36 @@
+import sys
+import os
+
+BASEDIR = os.path.dirname(os.path.abspath(__file__))
+
+if sys.platform == 'win32':
+    # add local folder to path
+    lib = os.path.join(BASEDIR, '_lib', 'magic')
+    envpath = os.environ['PATH']
+    os.environ['PATH'] = ';'.join([lib, envpath])
+
+try:
+    import magic
+    AUTODETECT = True
+except ImportError, e:
+    if sys.platform == 'darwin':
+        raise ImportError('Please install libmagic')
+    AUTODETECT = False
+
+
+class Detector(object):
+    def __init__(self):
+        if AUTODETECT:
+            magic_db = os.path.join(BASEDIR, '_lib', 'magic', 'db', 'magic.mgc')
+            self.magic = magic.Magic(magic_file=magic_db, mime_encoding=True)
+        else:
+            self.magic = False
+
+
+    def detect(self, filepath):
+        if self.magic:
+            encoding = self.magic.from_file(filepath)
+            return encoding
+        return None
+
+

pandasqt/_lib/magic/libgcc_s_dw2-1.dll

@@ -4,9 +4,9 @@
 from encodings.aliases import aliases as _encodings
 
 import pandas
-from chardet.universaldetector import UniversalDetector
 
 from pandasqt.compat import Qt, QtCore, QtGui, Slot, Signal
+from pandasqt.encoding import Detector
 from pandasqt.models.DataFrameModel import DataFrameModel
 from pandasqt.views.CustomDelegates import DtypeComboDelegate
 from pandasqt.views._ui import icons_rc
@@ -200,6 +200,7 @@ def __init__(self, parent=None):
         self._filename = None
         self._delimiter = None
         self._header = None
+        self._detector = Detector()
         self._initUI()
 
     def _initUI(self):
@@ -352,19 +353,18 @@ def _guessEncoding(self, path):
 
         """
         if os.path.exists(path) and path.lower().endswith('csv'):
-            encodingDetector = UniversalDetector()
-            with open(path, 'r') as fp:
-                for line in fp:
-                    encodingDetector.feed(line)
-                    if encodingDetector.done:
-                        break
-            encodingDetector.close()
-            result = encodingDetector.result['encoding']
-            result = result.replace('-','_')
-
-            self._encodingKey = _calculateEncodingKey(result)
-            if self._encodingKey:
-                index = self._encodingComboBox.findText(result.upper())
+            encoding = self._detector.detect(path)
+
+            if encoding is not None:
+                if encoding.startswith('utf'):
+                    encoding = encoding.replace('-', '')
+                encoding = encoding.replace('-','_')
+
+                viewValue = _encodings.get(encoding)
+
+                self._encodingKey = encoding
+
+                index = self._encodingComboBox.findText(viewValue.upper())
                 self._encodingComboBox.setCurrentIndex(index)
 
     @Slot('int')

pandasqt/_lib/magic/libmagic-1.dll

@@ -53,7 +53,7 @@ def run_tests(self):
         errcode = pytest.main(self.test_args)
         sys.exit(errcode)
 
-tests_require = ['pandas >= 0.15.2', 'pyside', 'pytest', 'pytest-cov', 'pytest-qt']
+tests_require = ['pandas >= 0.15.2', 'pyside', 'pytest', 'pytest-cov', 'pytest-qt', 'python-magic==0.4.6']
 setup(
     name='pandas-qt',
     version=__version__,
@@ -62,7 +62,7 @@ def run_tests(self):
     namespace_packages = ['pandasqt'],
     author='Matthias Ludwig',
     tests_require=tests_require,
-    install_requires=['pandas>=0.15.1', 'chardet', 'pytest', 'pytest-qt==1.2.2', 'pytest-cov'],
+    install_requires=['pandas>=0.15.1', 'pytest', 'pytest-qt==1.2.2', 'pytest-cov', 'python-magic==0.4.6'],
     cmdclass={'test': PyTest},
     author_email='m.Ludwig@datalyze-solutions.com',
     description='catches exceptions inside qt applications and writes them to a message box and into a log file',