replace printable for try/except utf-8

.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: python
 sudo: false
 env:

qiita_db/metadata_template/test/test_util.py

@@ -147,12 +147,23 @@ def test_load_template_to_dataframe_lowercase(self):
         exp.rename(columns={"str_column": "str_CoLumn"}, inplace=True)
         assert_frame_equal(obs, exp)
 
-    def test_load_template_to_dataframe_non_utf8(self):
+    def test_load_template_to_dataframe_non_utf8_error(self):
         bad = EXP_SAMPLE_TEMPLATE.replace('Test Sample 2', 'Test Sample\x962')
         with self.assertRaises(ValueError):
             qdb.metadata_template.util.load_template_to_dataframe(
                 StringIO(bad))
 
+    def test_load_template_to_dataframe_non_utf8(self):
+        replace = EXP_SAMPLE_TEMPLATE.replace(
+            'Test Sample 2', u'Test Sample\x962')
+        qdb.metadata_template.util.load_template_to_dataframe(
+            StringIO(replace))
+        # setting back
+        replace = EXP_SAMPLE_TEMPLATE.replace(
+            u'Test Sample\x962', 'Test Sample 2')
+        qdb.metadata_template.util.load_template_to_dataframe(
+            StringIO(replace))
+
     def test_load_template_to_dataframe_typechecking(self):
         obs = qdb.metadata_template.util.load_template_to_dataframe(
             StringIO(EXP_SAMPLE_TEMPLATE_LAT_ALL_INT))

qiita_db/metadata_template/util.py

@@ -9,7 +9,7 @@
 from __future__ import division
 from future.utils import PY3, viewitems
 from six import StringIO
-from string import printable
+from collections import defaultdict
 
 import pandas as pd
 import numpy as np
@@ -103,17 +103,16 @@ def load_template_to_dataframe(fn, index='sample_name'):
     # Load in file lines
     holdfile = None
     with open_file(fn, mode='U') as f:
-        errors = {}
+        errors = defaultdict(list)
         holdfile = f.readlines()
-        # here we are checking for non printable chars AKA non UTF-8 chars
+        # here we are checking for non UTF-8 chars
         for row, line in enumerate(holdfile):
             for col, block in enumerate(line.split('\t')):
-                tblock = ''.join([c for c in block if c in printable])
-                if len(block) != len(tblock):
-                    tblock = ''.join([c if c in printable else '🐾'
-                                      for c in block])
-                    if tblock not in errors:
-                        errors[tblock] = []
+                try:
+                    tblock = block.encode('utf-8')
+                except UnicodeDecodeError:
+                    tblock = unicode(block, errors='replace')
+                    tblock = tblock.replace(u'\ufffd', '🐾')
                     errors[tblock].append('(%d, %d)' % (row, col))
         if bool(errors):
             raise ValueError(