Merge pull request #2261 from antgonza/upgrading-software

upgrading software

Author: josenavas

qiita_db/metadata_template/util.py

@@ -14,7 +14,6 @@
 import pandas as pd
 import numpy as np
 import warnings
-from skbio.io.util import open_file
 from skbio.util import find_duplicates
 
 import qiita_db as qdb
@@ -102,7 +101,7 @@ def load_template_to_dataframe(fn, index='sample_name'):
     """
     # Load in file lines
     holdfile = None
-    with open_file(fn, mode='U') as f:
+    with qdb.util.open_file(fn, mode='U') as f:
         errors = defaultdict(list)
         holdfile = f.readlines()
         # here we are checking for non UTF-8 chars
@@ -333,7 +332,7 @@ def looks_like_qiime_mapping_file(fp):
     some other different column.
     """
     first_line = None
-    with open_file(fp, mode='U') as f:
+    with qdb.util.open_file(fp, mode='U') as f:
         first_line = f.readline()
     if not first_line:
         return False

qiita_db/search.py

@@ -69,6 +69,7 @@
 
 import pandas as pd
 from future.utils import viewitems
+from future.builtins import str
 
 from qiita_core.qiita_settings import qiita_config
 import qiita_db as qdb

qiita_db/test/test_util.py

@@ -7,14 +7,15 @@
 # -----------------------------------------------------------------------------
 
 from unittest import TestCase, main
-from tempfile import mkstemp
+from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile
 from os import close, remove, makedirs, mkdir
 from os.path import join, exists, basename
 from shutil import rmtree
 from datetime import datetime
 from functools import partial
 from string import punctuation
-
+import h5py
+from six import StringIO, BytesIO
 import pandas as pd
 
 from qiita_core.util import qiita_test_checker
@@ -869,5 +870,77 @@ def test_get_artifacts_information(self):
         self.assertItemsEqual(obs, exp)
 
 
+class TestFilePathOpening(TestCase):
+    """Tests adapted from scikit-bio's skbio.io.util tests"""
+    def test_is_string_or_bytes(self):
+        self.assertTrue(qdb.util._is_string_or_bytes('foo'))
+        self.assertTrue(qdb.util._is_string_or_bytes(u'foo'))
+        self.assertTrue(qdb.util._is_string_or_bytes(b'foo'))
+        self.assertFalse(qdb.util._is_string_or_bytes(StringIO('bar')))
+        self.assertFalse(qdb.util._is_string_or_bytes([1]))
+
+    def test_file_closed(self):
+        """File gets closed in decorator"""
+        f = NamedTemporaryFile('r')
+        filepath = f.name
+        with qdb.util.open_file(filepath) as fh:
+            pass
+        self.assertTrue(fh.closed)
+
+    def test_file_closed_harder(self):
+        """File gets closed in decorator, even if exceptions happen."""
+        f = NamedTemporaryFile('r')
+        filepath = f.name
+        try:
+            with qdb.util.open_file(filepath) as fh:
+                raise TypeError
+        except TypeError:
+            self.assertTrue(fh.closed)
+        else:
+            # If we're here, no exceptions have been raised inside the
+            # try clause, so the context manager swallowed them. No
+            # good.
+            raise Exception("`open_file` didn't propagate exceptions")
+
+    def test_filehandle(self):
+        """Filehandles slip through untouched"""
+        with TemporaryFile('r') as fh:
+            with qdb.util.open_file(fh) as ffh:
+                self.assertTrue(fh is ffh)
+            # And it doesn't close the file-handle
+            self.assertFalse(fh.closed)
+
+    def test_StringIO(self):
+        """StringIO (useful e.g. for testing) slips through."""
+        f = StringIO("File contents")
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_BytesIO(self):
+        """BytesIO (useful e.g. for testing) slips through."""
+        f = BytesIO(b"File contents")
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_hdf5IO(self):
+        f = h5py.File('test', driver='core', backing_store=False)
+        with qdb.util.open_file(f) as fh:
+            self.assertTrue(fh is f)
+
+    def test_hdf5IO_open(self):
+        name = None
+        with NamedTemporaryFile(delete=False) as fh:
+            name = fh.name
+            fh.close()
+
+            h5file = h5py.File(name, 'w')
+            h5file.close()
+
+            with qdb.util.open_file(name) as fh_inner:
+                self.assertTrue(isinstance(fh_inner, h5py.File))
+
+        remove(name)
+
+
 if __name__ == '__main__':
     main()

qiita_db/util.py

@@ -54,6 +54,9 @@
 from json import dumps
 from datetime import datetime
 from itertools import chain
+from contextlib import contextmanager
+from future.builtins import bytes, str
+import h5py
 
 from qiita_core.exceptions import IncompetentQiitaDeveloperError
 import qiita_db as qdb
@@ -1481,3 +1484,67 @@ def get_artifacts_information(artifact_ids, only_biom=True):
                     'files': filepaths})
 
             return results
+
+
+def _is_string_or_bytes(s):
+    """Returns True if input argument is string (unicode or not) or bytes.
+    """
+    return isinstance(s, str) or isinstance(s, bytes)
+
+
+def _get_filehandle(filepath_or, *args, **kwargs):
+    """Open file if `filepath_or` looks like a string/unicode/bytes, else
+    pass through.
+    """
+    if _is_string_or_bytes(filepath_or):
+        if h5py.is_hdf5(filepath_or):
+            fh, own_fh = h5py.File(filepath_or, *args, **kwargs), True
+        else:
+            fh, own_fh = open(filepath_or, *args, **kwargs), True
+    else:
+        fh, own_fh = filepath_or, False
+    return fh, own_fh
+
+
+@contextmanager
+def open_file(filepath_or, *args, **kwargs):
+    """Context manager, like ``open``, but lets file handles and file like
+    objects pass untouched.
+
+    It is useful when implementing a function that can accept both
+    strings and file-like objects (like numpy.loadtxt, etc).
+
+    This method differs slightly from scikit-bio's implementation in that it
+    handles HDF5 files appropriately.
+
+    Parameters
+    ----------
+    filepath_or : str/bytes/unicode string or file-like
+         If string, file to be opened using ``h5py.File`` if the file is an
+         HDF5 file, otherwise builtin ``open`` will be used. If it is not a
+         string, the object is just returned untouched.
+
+    Other parameters
+    ----------------
+    args, kwargs : tuple, dict
+        When `filepath_or` is a string, any extra arguments are passed
+        on to the ``open`` builtin.
+
+    Examples
+    --------
+    >>> with open_file('filename') as f:  # doctest: +SKIP
+    ...     pass
+    >>> fh = open('filename')             # doctest: +SKIP
+    >>> with open_file(fh) as f:          # doctest: +SKIP
+    ...     pass
+    >>> fh.closed                         # doctest: +SKIP
+    False
+    >>> fh.close()                        # doctest: +SKIP
+
+    """
+    fh, own_fh = _get_filehandle(filepath_or, *args, **kwargs)
+    try:
+        yield fh
+    finally:
+        if own_fh:
+            fh.close()

qiita_pet/handlers/api_proxy/artifact.py

@@ -10,8 +10,8 @@
 from json import dumps
 
 from future.utils import viewitems
+from itertools import chain
 from moi import r_client
-from skbio.util import flatten
 
 from qiita_core.util import execute_as_transaction
 from qiita_core.qiita_settings import qiita_config
@@ -103,8 +103,8 @@ def artifact_get_prep_req(user_id, artifact_ids):
         if access_error:
             return access_error
 
-        samples[aid] = flatten(
-            [pt.keys() for pt in Artifact(aid).prep_templates])
+        samples[aid] = list(chain(
+            *[pt.keys() for pt in Artifact(aid).prep_templates]))
 
     return {'status': 'success', 'msg': '', 'data': samples}
 
@@ -183,6 +183,7 @@ def artifact_post_req(user_id, filepaths, artifact_type, name,
         uploads_path = get_mountpoint('uploads')[0][1]
         path_builder = partial(join, uploads_path, str(study_id))
         cleaned_filepaths = {}
+
         for ftype, file_list in viewitems(filepaths):
             # JavaScript sends us this list as a comma-separated list
             for fp in file_list.split(','):

qiita_ware/ebi.py

@@ -22,10 +22,9 @@
 
 from qiita_core.qiita_settings import qiita_config
 from qiita_ware.exceptions import EBISubmissionError
-from qiita_ware.util import open_file
 from qiita_db.logger import LogEntry
 from qiita_db.ontology import Ontology
-from qiita_db.util import convert_to_id, get_mountpoint
+from qiita_db.util import convert_to_id, get_mountpoint, open_file
 from qiita_db.artifact import Artifact
 
 

qiita_ware/test/test_util.py

@@ -10,17 +10,11 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-import os
 from unittest import TestCase, main
-import tempfile
-
-import h5py
 import numpy as np
-from six import StringIO, BytesIO
 
 from qiita_db.metadata_template.prep_template import PrepTemplate
-from qiita_ware.util import (per_sample_sequences, open_file,
-                             _is_string_or_bytes)
+from qiita_ware.util import per_sample_sequences
 
 
 def mock_sequence_iter(items):
@@ -111,78 +105,6 @@ def test_dataframe_from_template(self):
             u'study_center'})
 
 
-class TestFilePathOpening(TestCase):
-    """Tests adapted from scikit-bio's skbio.io.util tests"""
-    def test_is_string_or_bytes(self):
-        self.assertTrue(_is_string_or_bytes('foo'))
-        self.assertTrue(_is_string_or_bytes(u'foo'))
-        self.assertTrue(_is_string_or_bytes(b'foo'))
-        self.assertFalse(_is_string_or_bytes(StringIO('bar')))
-        self.assertFalse(_is_string_or_bytes([1]))
-
-    def test_file_closed(self):
-        """File gets closed in decorator"""
-        f = tempfile.NamedTemporaryFile('r')
-        filepath = f.name
-        with open_file(filepath) as fh:
-            pass
-        self.assertTrue(fh.closed)
-
-    def test_file_closed_harder(self):
-        """File gets closed in decorator, even if exceptions happen."""
-        f = tempfile.NamedTemporaryFile('r')
-        filepath = f.name
-        try:
-            with open_file(filepath) as fh:
-                raise TypeError
-        except TypeError:
-            self.assertTrue(fh.closed)
-        else:
-            # If we're here, no exceptions have been raised inside the
-            # try clause, so the context manager swallowed them. No
-            # good.
-            raise Exception("`open_file` didn't propagate exceptions")
-
-    def test_filehandle(self):
-        """Filehandles slip through untouched"""
-        with tempfile.TemporaryFile('r') as fh:
-            with open_file(fh) as ffh:
-                self.assertTrue(fh is ffh)
-            # And it doesn't close the file-handle
-            self.assertFalse(fh.closed)
-
-    def test_StringIO(self):
-        """StringIO (useful e.g. for testing) slips through."""
-        f = StringIO("File contents")
-        with open_file(f) as fh:
-            self.assertTrue(fh is f)
-
-    def test_BytesIO(self):
-        """BytesIO (useful e.g. for testing) slips through."""
-        f = BytesIO(b"File contents")
-        with open_file(f) as fh:
-            self.assertTrue(fh is f)
-
-    def test_hdf5IO(self):
-        f = h5py.File('test', driver='core', backing_store=False)
-        with open_file(f) as fh:
-            self.assertTrue(fh is f)
-
-    def test_hdf5IO_open(self):
-        name = None
-        with tempfile.NamedTemporaryFile(delete=False) as fh:
-            name = fh.name
-            fh.close()
-
-            h5file = h5py.File(name, 'w')
-            h5file.close()
-
-            with open_file(name) as fh_inner:
-                self.assertTrue(isinstance(fh_inner, h5py.File))
-
-        os.remove(name)
-
-
 # comment indicates the expected random value
 sequences = [
     ('a_1', 'AATTGGCC-a1'),  # 2, 3624216819017203053