Additional Processing Command added to Software_Command

.travis.yml

@@ -27,7 +27,7 @@ install:
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip
-  - pip install sphinx sphinx-bootstrap-theme nose-timer codecov
+  - pip install sphinx sphinx-bootstrap-theme nose-timer codecov 'Click==6.7'
   - 'echo "backend: Agg" > matplotlibrc'
   - git clone https://github.com/nicolasff/webdis
   - pushd webdis

qiita_db/analysis.py

@@ -838,6 +838,8 @@ def build_files(self, merge_duplicated_sample_ids):
             # are going to create
             rename_dup_samples = False
             grouped_samples = {}
+
+            addtl_processing_cmd = {}
             for aid, asamples in viewitems(samples):
                 # find the artifat info, [0] there should be only 1 info
                 ainfo = [bi for bi in bioms_info
@@ -847,6 +849,11 @@ def build_files(self, merge_duplicated_sample_ids):
 
                 label = "%s || %s" % (data_type, ainfo['algorithm'])
                 if label not in grouped_samples:
+                    aparams = qdb.artifact.Artifact(aid).processing_parameters
+                    if aparams is not None:
+                        fam = aparams.command.addtl_processing_cmd
+                        if fam is not None:
+                            addtl_processing_cmd[label] = fam
                     grouped_samples[label] = []
                 grouped_samples[label].append((aid, asamples))
             # 2. if rename_dup_samples is still False, make sure that we don't
@@ -872,6 +879,11 @@ def build_files(self, merge_duplicated_sample_ids):
             biom_files = self._build_biom_tables(
                 grouped_samples, rename_dup_samples)
 
+            # if values for addtl_processing_cmd were found, return them as
+            # well, if only to allow for testing.
+            if addtl_processing_cmd:
+                return (biom_files, addtl_processing_cmd)
+
             return biom_files
 
     def _build_biom_tables(self, grouped_samples, rename_dup_samples=False):
@@ -934,16 +946,22 @@ def _build_biom_tables(self, grouped_samples, rename_dup_samples=False):
                                        "analysis due to rarefaction level")
 
                 # write out the file
+                # data_type and algorithm values become part of the file
+                # name(s).
                 data_type = sub('[^0-9a-zA-Z]+', '', data_type)
                 algorithm = sub('[^0-9a-zA-Z]+', '', algorithm)
                 info = "%s_%s" % (data_type, algorithm)
                 fn = "%d_analysis_%s.biom" % (self._id, info)
                 biom_fp = join(base_fp, fn)
+                # save final biom here
                 with biom_open(biom_fp, 'w') as f:
                     new_table.to_hdf5(
                         f, "Generated by Qiita, analysis id: %d, info: %s" % (
                             self._id, label))
                 biom_files.append((data_type, biom_fp))
+
+        # return the biom files, either with or without needed tree, to
+        # the user.
         return biom_files
 
     def _build_mapping_file(self, samples, rename_dup_samples=False):

qiita_db/software.py

@@ -27,6 +27,18 @@ class Command(qdb.base.QiitaObject):
 
     Attributes
     ----------
+    active
+    addtl_processing_cmd
+    analysis_only
+    default_parameter_sets
+    description
+    merging_scheme
+    name
+    naming_order
+    optional_parameters
+    outputs
+    parameters
+    required_parameters
     software
     name
     description
@@ -35,7 +47,16 @@ class Command(qdb.base.QiitaObject):
 
     Methods
     -------
+    _check_id
+    activate
+
+    Class Methods
+    -------------
     create
+    exists
+    get_commands_by_input_type(cls, artifact_types, active_only=True,
+    get_html_generator(cls, artifact_type):
+    get_validator(cls, artifact_type):
 
     See Also
     --------
@@ -445,6 +466,22 @@ def name(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
+    @property
+    def addtl_processing_cmd(self):
+        """Additional processing commands required for merging
+
+        Returns
+        -------
+        str
+            Returns the additional processing command for merging
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT addtl_processing_cmd
+                     FROM qiita.software_command
+                     WHERE command_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
     @property
     def description(self):
         """The description of the command

qiita_db/support_files/patches/67.sql

@@ -0,0 +1,6 @@
+-- October 6, 2018
+-- add addtl_processing_cmd column to record additional information required to merge some BIOMS.
+
+ALTER TABLE qiita.software_command ADD addtl_processing_cmd varchar;
+
+COMMENT ON COLUMN qiita.software_command.addtl_processing_cmd IS 'Store information on additional processes to merge BIOMs, if any.';

qiita_db/support_files/qiita-db.dbs

@@ -1307,6 +1307,9 @@
 			<column name="ignore_parent_command" type="bool" jt="-7" mandatory="y" >
 				<defo>false</defo>
 			</column>
+			<column name="addtl_processing_cmd" type="varchar" jt="12">
+                            <comment><![CDATA[Store information on additional processes to merge BIOMs, if any.]]></comment>
+                        </column>
 			<index name="pk_soft_command" unique="PRIMARY_KEY" >
 				<column name="command_id" />
 			</index>
@@ -1854,4 +1857,4 @@ ALTER TABLE oauth_software ADD CONSTRAINT fk_oauth_software FOREIGN KEY ( client
 ]]></string>
 		</script>
 	</layout>
-</project>
+</project>

qiita_db/support_files/qiita-db.html

@@ -1911,6 +1911,8 @@
   <use id='nn' x='2162' y='1332' xlink:href='#nn'/><a xlink:href='#software_command.active'><text x='2178' y='1342'>active</text><title>active bool not null default &#039;True&#039;</title></a>
   <use id='nn' x='2162' y='1347' xlink:href='#nn'/><a xlink:href='#software_command.is_analysis'><text x='2178' y='1357'>is_analysis</text><title>is_analysis bool not null default &#039;False&#039;</title></a>
   <use id='nn' x='2162' y='1362' xlink:href='#nn'/><a xlink:href='#software_command.ignore_parent_command'><text x='2178' y='1372'>ignore_parent_command</text><title>ignore_parent_command bool not null default false</title></a>
+  <use id='nn' x='2162' y='1377' xlink:href='#nn'/><a xlink:href='#software_command.addtl_processing_cmd'><text x='2178' y='1387'>addtl_processing_cmd</text><title>addtl_processing_cmd varchar
+Store information on additional processes to merge BIOMs, if any.</title></a>
 
 <!-- ============= Table 'processing_job_validator' ============= -->
 <rect class='table' x='2160' y='1403' width='150' height='90' rx='7' ry='7' />
@@ -5722,6 +5724,12 @@
 		<td> bool   DEFO false </td>
 		<td>  </td>
 	</tr>
+	<tr>
+		<td>*</td>
+		<td><a name='software_command.addtl_processing_cmd'>addtl&#095;processing&#095;cmd</a></td>
+		<td> varchar   </td>
+		<td>Store information on additional processes to merge BIOMs, if any.</td>
+	</tr>
 <tr><th colspan='4'><b>Indexes</b></th></tr>
 	<tr>		<td>Pk</td><td>pk&#095;soft&#095;command</td>
 		<td> ON command&#095;id</td>
@@ -6176,5 +6184,4 @@
 	</tr>
 </tbody>
 </table>
-
-</body></html>
+</body></html>

qiita_db/test/test_analysis.py

@@ -76,7 +76,7 @@ def _create_analyses_with_samples(self, user='demo@microbio.me',
         Parameters
         ----------
         user : qiita_db.user.User, optional
-            The user email to attach the analysis. Default: demo@microbio.me
+            The user email to attach to the analysis. Default: demo@microbio.me
         merge : bool, optional
             Merge duplicated ids or not
 
@@ -91,14 +91,17 @@ def _create_analyses_with_samples(self, user='demo@microbio.me',
         """
         user = qdb.user.User(user)
         dflt_analysis = user.default_analysis
+
         dflt_analysis.add_samples(
             {4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196',
                  '1.SKM9.640192', '1.SKM4.640180']})
+
         new = qdb.analysis.Analysis.create(
             user, "newAnalysis", "A New Analysis", from_default=True,
             merge_duplicated_sample_ids=merge)
 
         self._wait_for_jobs(new)
+
         return new
 
     def test_lock_samples(self):
@@ -512,6 +515,39 @@ def test_build_files(self):
                '1.SKB8.640193', '1.SKB7.640196']
         self.assertItemsEqual(biom_ids, exp)
 
+    def test_build_files_addtl_processing_cmd(self):
+        tmp = qdb.artifact.Artifact(4).processing_parameters.command
+        cmd_id = tmp.id
+
+        # set a known artifact's additional processing command
+        # to a known value. Then test for it.
+        with qdb.sql_connection.TRN:
+            sql = """UPDATE qiita.software_command
+                     SET addtl_processing_cmd = 'ls'
+                     WHERE command_id = %s""" % cmd_id
+            qdb.sql_connection.TRN.add(sql, [cmd_id])
+            qdb.sql_connection.TRN.execute()
+
+        analysis = self._create_analyses_with_samples()
+        results = npt.assert_warns(
+            qdb.exceptions.QiitaDBWarning, analysis.build_files, False)
+
+        # if build_files used additional processing commands, it will
+        # return a tuple, where the second element is a dictionary of
+        # the commands used.
+        # biom_tables = results[0]
+        addtl_processing_cmds = results[1]
+        for k in addtl_processing_cmds:
+            self.assertItemsEqual('ls', addtl_processing_cmds[k])
+
+        # cleanup (assume command was NULL previously)
+        with qdb.sql_connection.TRN:
+            sql = """UPDATE qiita.software_command
+                     SET addtl_processing_cmd = NULL
+                     WHERE command_id = %s""" % cmd_id
+            qdb.sql_connection.TRN.add(sql, [cmd_id])
+            qdb.sql_connection.TRN.execute()
+
     def test_build_files_merge_duplicated_sample_ids(self):
         user = qdb.user.User("demo@microbio.me")
         dflt_analysis = user.default_analysis

qiita_db/test/test_software.py

@@ -128,6 +128,30 @@ def test_name(self):
         self.assertEqual(qdb.software.Command(1).name, "Split libraries FASTQ")
         self.assertEqual(qdb.software.Command(2).name, "Split libraries")
 
+    def test_addtl_processing_cmd(self):
+        # initial test
+        self.assertEqual(qdb.software.Command(1).addtl_processing_cmd, None)
+
+        # modify table directly, in order to test method
+        with qdb.sql_connection.TRN:
+            sql = """UPDATE qiita.software_command
+                     SET addtl_processing_cmd = 'ls'
+                     WHERE command_id = 1"""
+            qdb.sql_connection.TRN.add(sql, [1])
+            qdb.sql_connection.TRN.execute()
+
+        # test method returns 'ls'
+        self.assertEqual(qdb.software.Command(1).addtl_processing_cmd, 'ls')
+
+        # clean up table
+        with qdb.sql_connection.TRN:
+            sql = """UPDATE qiita.software_command
+                     SET addtl_processing_cmd = NULL
+                     WHERE command_id = 1"""
+            qdb.sql_connection.TRN.add(sql, [1])
+            qdb.sql_connection.TRN.execute()
+        self.assertEqual(qdb.software.Command(1).addtl_processing_cmd, None)
+
     def test_description(self):
         self.assertEqual(
             qdb.software.Command(1).description,