Release candidate November 1st, 2017

.travis.yml

@@ -28,7 +28,7 @@ install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
   - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8
-    pyzmq networkx pyparsing natsort mock future libgfortran seaborn nltk
+    pyzmq 'networkx<2.0' pyparsing natsort mock future libgfortran seaborn nltk
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip

qiita_db/analysis.py

@@ -178,7 +178,7 @@ def create(cls, owner, name, description, from_default=False,
                     'analysis': a_id,
                     'merge_dup_sample_ids': merge_duplicated_sample_ids})
             job = qdb.processing_job.ProcessingJob.create(
-                owner, params)
+                owner, params, True)
             sql = """INSERT INTO qiita.analysis_processing_job
                         (analysis_id, processing_job_id)
                      VALUES (%s, %s)"""
@@ -429,15 +429,17 @@ def mapping_file(self):
 
         Returns
         -------
-        str or None
-            full filepath to the mapping file or None if not generated
+        int or None
+            The filepath id of the analysis mapping file or None
+            if not generated
         """
-        fp = [fp for _, fp, fp_type in qdb.util.retrieve_filepaths(
-            "analysis_filepath", "analysis_id", self._id)
-            if fp_type == 'plain_text']
+        fp = [fp_id
+              for fp_id, _, fp_type in qdb.util.retrieve_filepaths(
+                "analysis_filepath", "analysis_id", self._id)
+              if fp_type == 'plain_text']
 
         if fp:
-            # returning the actual path vs. an array
+            # returning the actual filepath id vs. an array
             return fp[0]
         else:
             return None

qiita_db/handlers/analysis.py

@@ -59,7 +59,8 @@ def get(self, analysis_id):
         """
         with qdb.sql_connection.TRN:
             a = _get_analysis(analysis_id)
-            mf_fp = a.mapping_file
+            mf_fp = qdb.util.get_filepath_information(
+                a.mapping_file)['fullpath']
             response = None
             if mf_fp is not None:
                 df = qdb.metadata_template.util.load_template_to_dataframe(

qiita_db/handlers/processing_job.py

@@ -163,7 +163,7 @@ def post(self):
         params = qdb.software.Parameters.load(cmd, json_str=params_dict)
 
         job = qdb.processing_job.ProcessingJob.create(
-            qdb.user.User(user), params)
+            qdb.user.User(user), params, True)
 
         if status:
             job._set_status(status)

qiita_db/handlers/tests/test_artifact.py

@@ -103,8 +103,8 @@ def test_get_artifact(self):
             'prep_information': [],
             'study': None,
             'analysis': 1,
-            'processing_parameters': {'biom_table': 8, 'depth': 9000,
-                                      'subsample_multinomial': False},
+            'processing_parameters': {'biom_table': '8', 'depth': '9000',
+                                      'subsample_multinomial': 'False'},
             'files': exp_fps}
         obs = loads(obs.body)
         # The timestamp is genreated at patch time, so we can't check for it

qiita_db/metadata_template/test/test_prep_template.py

@@ -887,9 +887,7 @@ def test_create(self):
     def test_create_already_prefixed_samples(self):
         """Creates a new PrepTemplate"""
         fp_count = qdb.util.get_count('qiita.filepath')
-        pt = npt.assert_warns(
-            qdb.exceptions.QiitaDBWarning,
-            qdb.metadata_template.prep_template.PrepTemplate.create,
+        pt = qdb.metadata_template.prep_template.PrepTemplate.create(
             self.metadata_prefixed, self.test_study, self.data_type)
         self._common_creation_checks(pt, fp_count)
 

qiita_db/metadata_template/test/test_sample_template.py

@@ -1084,9 +1084,7 @@ def test_create_str_prefixes(self):
 
     def test_create_already_prefixed_samples(self):
         """Creates a new SampleTemplate with the samples already prefixed"""
-        st = npt.assert_warns(
-            qdb.exceptions.QiitaDBWarning,
-            qdb.metadata_template.sample_template.SampleTemplate.create,
+        st = qdb.metadata_template.sample_template.SampleTemplate.create(
             self.metadata_prefixed, self.new_study)
         new_id = self.new_study.id
         # The returned object has the correct id

qiita_db/metadata_template/test/test_util.py

@@ -8,6 +8,7 @@
 
 from six import StringIO
 from unittest import TestCase, main
+import warnings
 
 import numpy.testing as npt
 import pandas as pd
@@ -36,12 +37,31 @@ def test_prefix_sample_names_with_id(self):
         }
         exp_df = pd.DataFrame.from_dict(exp_metadata_dict, orient='index',
                                         dtype=str)
-        qdb.metadata_template.util.prefix_sample_names_with_id(
-            self.metadata_map, 1)
+        with warnings.catch_warnings(record=True) as warn:
+            qdb.metadata_template.util.prefix_sample_names_with_id(
+                self.metadata_map, 1)
+            self.assertEqual(len(warn), 0)
         self.metadata_map.sort_index(inplace=True)
         exp_df.sort_index(inplace=True)
         assert_frame_equal(self.metadata_map, exp_df)
 
+        # test that it only prefixes the samples that are needed
+        metadata_dict = {
+            'Sample1': {'int_col': 1, 'float_col': 2.1, 'str_col': 'str1'},
+            '1.Sample2': {'int_col': 2, 'float_col': 3.1, 'str_col': '200'},
+            'Sample3': {'int_col': 3, 'float_col': 3, 'str_col': 'string30'},
+        }
+        metadata_map = pd.DataFrame.from_dict(
+            metadata_dict, orient='index', dtype=str)
+        with warnings.catch_warnings(record=True) as warn:
+            qdb.metadata_template.util.prefix_sample_names_with_id(
+                metadata_map, 1)
+            self.assertEqual(len(warn), 1)
+            self.assertEqual(str(warn[0].message), 'Some of the samples were '
+                             'already prefixed with the study id.')
+        metadata_map.sort_index(inplace=True)
+        assert_frame_equal(metadata_map, exp_df)
+
     def test_load_template_to_dataframe(self):
         obs = qdb.metadata_template.util.load_template_to_dataframe(
             StringIO(EXP_SAMPLE_TEMPLATE))

qiita_db/metadata_template/util.py

@@ -34,29 +34,25 @@ def prefix_sample_names_with_id(md_template, study_id):
     study_id : int
         The study to which the metadata belongs to
     """
-    # Get all the prefixes of the index, defined as any string before a '.'
-    prefixes = {idx.split('.', 1)[0] for idx in md_template.index}
-    # If the samples have been already prefixed with the study id, the prefixes
-    # set will contain only one element and it will be the str representation
-    # of the study id
-    if len(prefixes) == 1 and prefixes.pop() == str(study_id):
-        # The samples were already prefixed with the study id
-        warnings.warn("Sample names were already prefixed with the study id.",
-                      qdb.exceptions.QiitaDBWarning)
-    else:
-        # Create a new pandas series in which all the values are the study_id
-        # and it is indexed as the metadata template
-        study_ids = pd.Series([str(study_id)] * len(md_template.index),
-                              index=md_template.index)
-        # Create a new column on the metadata template that includes the
-        # metadata template indexes prefixed with the study id
-        md_template['sample_name_with_id'] = (study_ids + '.' +
-                                              md_template.index.values)
-        md_template.index = md_template.sample_name_with_id
-        del md_template['sample_name_with_id']
-        # The original metadata template had the index column unnamed - remove
-        # the name of the index for consistency
-        md_template.index.name = None
+    # loop over the samples and prefix those that aren't prefixed
+    md_template['qiita_sample_name_with_id'] = pd.Series(
+        [idx if idx.split('.', 1)[0] == str(study_id)
+         else '%d.%s' % (study_id, idx)
+         for idx in md_template.index], index=md_template.index)
+
+    # get the rows that are gonna change
+    changes = len(md_template.index[
+        md_template['qiita_sample_name_with_id'] != md_template.index])
+    if changes != 0 and changes != len(md_template.index):
+        warnings.warn(
+            "Some of the samples were already prefixed with the study id.",
+            qdb.exceptions.QiitaDBWarning)
+
+    md_template.index = md_template.qiita_sample_name_with_id
+    del md_template['qiita_sample_name_with_id']
+    # The original metadata template had the index column unnamed -> remove
+    # the name of the index for consistency
+    md_template.index.name = None
 
 
 def load_template_to_dataframe(fn, index='sample_name'):