sample info file generate_files is now "smart"

qiita_db/metadata_template/base_metadata_template.py

@@ -696,7 +696,7 @@ def _common_delete_sample_steps(self, sample_names):
                 qdb.sql_connection.TRN.add(sql2, [sn, self.id])
             qdb.sql_connection.TRN.execute()
 
-            self.generate_files()
+            self.generate_files(samples=sample_names)
 
     def delete_column(self, column_name):
         """Delete `column_name` from info file
@@ -803,6 +803,13 @@ def _common_extend_steps(self, md_template):
         ----------
         md_template : DataFrame
             The metadata template file contents indexed by sample ids
+
+        Returns
+        -------
+        list of str
+            The new samples being added
+        list of str
+            The new columns being added
         """
         with qdb.sql_connection.TRN:
             # Check if we are adding new samples
@@ -887,6 +894,8 @@ def _common_extend_steps(self, md_template):
             # Execute all the steps
             qdb.sql_connection.TRN.execute()
 
+        return new_samples, new_cols
+
     @classmethod
     def exists(cls, obj_id):
         r"""Checks if already exists a MetadataTemplate for the provided object
@@ -1233,9 +1242,9 @@ def extend(self, md_template):
         with qdb.sql_connection.TRN:
             md_template = self._clean_validate_template(
                 md_template, self.study_id, current_columns=self.categories())
-            self._common_extend_steps(md_template)
+            new_samples, new_columns = self._common_extend_steps(md_template)
             self.validate(self.columns_restrictions)
-            self.generate_files()
+            self.generate_files(new_samples, new_columns)
 
     def _update(self, md_template):
         r"""Update values in the template
@@ -1245,6 +1254,13 @@ def _update(self, md_template):
         md_template : DataFrame
             The metadata template file contents indexed by samples ids
 
+        Returns
+        -------
+        set of str
+            The samples that were updated
+        set of str
+            The columns that were updated
+
         Raises
         ------
         QiitaDBError
@@ -1301,7 +1317,7 @@ def _update(self, md_template):
                     "There are no differences between the data stored in the "
                     "DB and the new data provided",
                     qdb.exceptions.QiitaDBWarning)
-                return
+                return set([]), set([])
 
             changed.index.names = ['sample_name', 'column']
             # the combination of np.where and boolean indexing produces
@@ -1312,7 +1328,9 @@ def _update(self, md_template):
             to_update = pd.DataFrame({'to': changed_to}, index=changed.index)
 
             new_columns = []
+            samples_updated = []
             for sid, df in to_update.groupby(level=0):
+                samples_updated.append(sid)
                 values = {k[1]: v for k, v in df.to_dict()['to'].iteritems()}
                 new_columns.extend(values.keys())
                 sql = """UPDATE qiita.{0}
@@ -1321,9 +1339,9 @@ def _update(self, md_template):
                             self._table_name(self._id))
                 qdb.sql_connection.TRN.add(sql, [dumps(values), sid])
 
-            new_columns = list(set(new_columns).union(set(self.categories())))
+            nc = list(set(new_columns).union(set(self.categories())))
             table_name = self._table_name(self.id)
-            values = '{"columns": %s}' % dumps(new_columns)
+            values = '{"columns": %s}' % dumps(nc)
             sql = """UPDATE qiita.{0}
                      SET sample_values = %s
                      WHERE sample_id = '{1}'""".format(
@@ -1332,6 +1350,8 @@ def _update(self, md_template):
 
             qdb.sql_connection.TRN.execute()
 
+        return set(samples_updated), set(new_columns)
+
     def update(self, md_template):
         r"""Update values in the template
 
@@ -1354,9 +1374,9 @@ def update(self, md_template):
             # Clean and validate the metadata template given
             new_map = self._clean_validate_template(
                 md_template, self.study_id, current_columns=self.categories())
-            self._update(new_map)
+            samples, columns = self._update(new_map)
             self.validate(self.columns_restrictions)
-            self.generate_files()
+            self.generate_files(samples, columns)
 
     def extend_and_update(self, md_template):
         """Performs the update and extend operations at once
@@ -1374,10 +1394,13 @@ def extend_and_update(self, md_template):
         with qdb.sql_connection.TRN:
             md_template = self._clean_validate_template(
                 md_template, self.study_id, current_columns=self.categories())
-            self._common_extend_steps(md_template)
-            self._update(md_template)
+            new_samples, new_columns = self._common_extend_steps(md_template)
+            samples, columns = self._update(md_template)
+            samples.update(new_samples)
+            columns.update(new_columns)
+
             self.validate(self.columns_restrictions)
-            self.generate_files()
+            self.generate_files(samples, columns)
 
     def update_category(self, category, samples_and_values):
         """Update an existing column

qiita_db/metadata_template/prep_template.py

@@ -480,8 +480,15 @@ def study_id(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
-    def generate_files(self):
+    def generate_files(self, samples=None, columns=None):
         r"""Generates all the files that contain data from this template
+
+        Parameters
+        ----------
+        samples : iterable of str, optional
+            The samples that were added/updated
+        columns : iterable of str, optional
+            The columns that were added/updated
         """
         with qdb.sql_connection.TRN:
             # figuring out the filepath of the prep template

qiita_db/metadata_template/sample_template.py

@@ -249,8 +249,15 @@ def can_be_extended(self, new_samples, new_columns):
         """
         return True, ""
 
-    def generate_files(self):
+    def generate_files(self, samples=None, columns=None):
         r"""Generates all the files that contain data from this template
+
+        Parameters
+        ----------
+        samples : iterable of str, optional
+            The samples that were added/updated
+        columns : iterable of str, optional
+            The columns that were added/updated
         """
         with qdb.sql_connection.TRN:
             # figuring out the filepath of the sample template
@@ -265,7 +272,14 @@ def generate_files(self):
 
             # generating all new QIIME mapping files
             for pt in qdb.study.Study(self._id).prep_templates():
-                pt.generate_files()
+                if samples is not None and samples and (
+                        columns is None or not columns):
+                    overlapping = set(samples) & set(pt.keys())
+                    # if the prep has no overlapping sample ids, we can skip
+                    # generationg the prep
+                    if not overlapping:
+                        continue
+                pt.generate_files(samples, columns)
 
     @property
     def ebi_sample_accessions(self):