WIP: Email status upgrade

qiita_db/artifact.py

@@ -618,13 +618,17 @@ def delete(cls, artifact_id):
                      WHERE artifact_id IN %s"""
             qdb.sql_connection.TRN.add(sql, [all_ids])
 
+            # do not move these files back to upload folder.
+            dnm = ['qtp-sequencing-validate-data.csv', 'feature-table.qza']
+
             # If the first artifact to be deleted, instance, doesn't have
             # parents and study is not None (None means is an analysis), we
             # move the files to the uploads folder. We also need
             # to nullify the column in the prep template table
             if not instance.parents and study is not None:
-                qdb.util.move_filepaths_to_upload_folder(
-                    study.id, filepaths)
+                qdb.util.move_filepaths_to_upload_folder(study.id,
+                                                         filepaths,
+                                                         do_not_move=dnm)
 
                 # there are cases that an artifact would not be linked to a
                 # study

qiita_db/processing_job.py

@@ -594,7 +594,7 @@ def create(cls, user, parameters, force=False):
                 if vals[0] == 'artifact':
                     artifact_info = parameters.values[pname]
                     # If the artifact_info is a list, then the artifact
-                    # still doesn't exists because the current job is part
+                    # still doesn't exist because the current job is part
                     # of a workflow, so we can't link
                     if not isinstance(artifact_info, list):
                         TTRN.add(sql, [artifact_info, job_id])
@@ -703,6 +703,62 @@ def status(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
+    def _notify_updated_status(self, value, error_msg):
+        ignored_software = ('artifact definition',)
+        ignored_commands = ('Validate', 'complete_job', 'release_validators')
+
+        # abort early conditions (don't send an email notification)
+        # tentatively accept the overhead of a function-call, even when a
+        # notification isn't sent, just to keep the logic clean and
+        # centralized.
+        if value == 'waiting':
+            # notification not needed.
+            return
+
+        if not self.user.info['receive_processing_job_emails']:
+            # notification not needed.
+            return
+
+        if self.command.software.name in ignored_software:
+            # notification not needed.
+            return
+
+        if self.command.name in ignored_commands:
+            # notification not needed.
+            return
+
+        # generate subject line
+        subject = 'Job status change: %s (%s)' % (self.command.name, self.id)
+
+        # generate message line
+        input_artifacts = self.input_artifacts()
+        if input_artifacts is None:
+            # this is an admin job. display command name and parameters
+            pass
+        else:
+            for artifact in input_artifacts:
+                if artifact.prep_templates is not None:
+                    # this is a processing job. display the study id as link,
+                    # prep ids, data_type, and command name.
+                    pass
+                elif artifact.analysis is not None:
+                    # this is an analysis job. display analysis id as link and
+                    # the command name.
+                    pass
+                else:
+                    raise ValueError("Unknown Condition")
+
+        message = ''
+
+        # append legacy message line
+        message += 'New status: %s' % (value)
+
+        if value == 'error' and error_msg is not None:
+            message += f'\n\nError:\n{error_msg}'
+
+        # send email
+        qdb.util.send_email(self.user.email, subject, message)
+
     def _set_status(self, value, error_msg=None):
         """Sets the status of the job
 
@@ -734,22 +790,7 @@ def _set_status(self, value, error_msg=None):
             new_status = qdb.util.convert_to_id(
                 value, "processing_job_status")
 
-            if value not in {'waiting'}:
-                if self.user.info['receive_processing_job_emails']:
-                    # skip if software is artifact definition
-                    ignore_software = ('artifact definition', )
-                    if self.command.software.name not in ignore_software:
-                        ignore_commands = ('Validate', 'complete_job',
-                                           'release_validators')
-                        if self.command.name not in ignore_commands:
-                            subject = 'Job status change: %s (%s)' % (
-                                self.command.name, self.id)
-                            message = 'New status: %s' % (value)
-
-                            if value == 'error' and error_msg is not None:
-                                message += f'\n\nError:\n{error_msg}'
-                            qdb.util.send_email(
-                                self.user.email, subject, message)
+            self._notify_updated_status(value, error_msg)
 
             sql = """UPDATE qiita.processing_job
                      SET processing_job_status_id = %s

qiita_db/test/test_util.py

@@ -384,25 +384,47 @@ def test_move_filepaths_to_upload_folder(self):
         # we are going to test the move_filepaths_to_upload_folder indirectly
         # by creating an artifact and deleting it. To accomplish this we need
         # to create a new prep info file, attach a biom with html_summary and
-        # then deleting it. However, we will do this twice to assure that
+        # then delete it. However, we will do this twice to assure that
         # there are no conflicts with this
         study_id = 1
         # creating the 2 sets of files for the 2 artifacts
         fd, seqs_fp1 = mkstemp(suffix='_seqs.fastq')
         close(fd)
+
         html_fp1 = mkdtemp()
         html_fp1 = join(html_fp1, 'support_files')
         mkdir(html_fp1)
         with open(join(html_fp1, 'index.html'), 'w') as fp:
             fp.write(">AAA\nAAA")
         fd, seqs_fp2 = mkstemp(suffix='_seqs.fastq')
         close(fd)
+
         html_fp2 = mkdtemp()
         html_fp2 = join(html_fp2, 'support_files')
         mkdir(html_fp2)
         with open(join(html_fp2, 'index.html'), 'w') as fp:
             fp.write(">AAA\nAAA")
 
+        # create an additional directory named 'more_files' and populate it
+        # with two files that shouldn't be moved back into the uploads
+        # directory when an artifact is deleted, and one that should.
+        more_files_fp = mkdtemp()
+        more_files_fp = join(more_files_fp, 'more_files')
+        mkdir(more_files_fp)
+        test_files = [(join(more_files_fp, x), 1) for x in
+                      ['qtp-sequencing-validate-data.csv',
+                       'feature-table.qza',
+                       'good_file.txt']]
+
+        # create the files
+        for file_name, _ in test_files:
+            with open(file_name, 'w') as fp:
+                fp.write("This is a test file.\n")
+
+        # verify that the files exist
+        for file_name, _ in test_files:
+            self.assertTrue(exists(file_name))
+
         # creating new prep info file
         metadata_dict = {
             'SKB8.640193': {'center_name': 'ANL',
@@ -422,7 +444,7 @@ def test_move_filepaths_to_upload_folder(self):
 
         # inserting artifact 1
         artifact1 = qdb.artifact.Artifact.create(
-            [(seqs_fp1, 1), (html_fp1, 'html_summary')], "FASTQ",
+            [(seqs_fp1, 1), (html_fp1, 'html_summary')] + test_files, "FASTQ",
             prep_template=pt1)
         # inserting artifact 2
         artifact2 = qdb.artifact.Artifact.create(
@@ -433,8 +455,23 @@ def test_move_filepaths_to_upload_folder(self):
         filepaths = artifact1.filepaths
         filepaths.extend(artifact2.filepaths)
 
-        # delete artifacts
+        # delete artifact 1
         qdb.artifact.Artifact.delete(artifact1.id)
+
+        # now that artifact 1 is deleted, confirm that all three test files
+        # are removed.
+        for file_name, _ in test_files:
+            self.assertFalse(exists(file_name))
+
+        # confirm that the 'bad_files' were not moved back into the uploads
+        # directory, but the 'good_file.txt' was.
+        uploads = qdb.util.get_files_from_uploads_folders("1")
+        bad_files = [basename(x[0]) for x in test_files if
+                     'good_file.txt' not in x[0]]
+        for _, some_path, _ in uploads:
+            self.assertFalse(basename(some_path) in bad_files)
+
+        # finish deleting artifacts
         qdb.artifact.Artifact.delete(artifact2.id)
 
         # now let's create another artifact with the same filenames that
@@ -455,7 +492,17 @@ def test_move_filepaths_to_upload_folder(self):
                                 str(study_id))
         for x in filepaths:
             self.assertFalse(exists(x['fp']))
-            new_fp = join(path_for_removal, basename(x['fp']))
+
+            f_name = basename(x['fp'])
+
+            if f_name in bad_files:
+                # skip the check for file-names in bad_files,
+                # because they were already checked above and they were
+                # already deleted.
+                continue
+
+            new_fp = join(path_for_removal, f_name)
+
             if x['fp_type'] == 'html_summary':
                 # The html summary gets removed, not moved
                 self.assertFalse(exists(new_fp))

qiita_db/util.py

@@ -940,7 +940,7 @@ def empty_trash_upload_folder(delete_files=True):
             qdb.sql_connection.TRN.execute()
 
 
-def move_filepaths_to_upload_folder(study_id, filepaths):
+def move_filepaths_to_upload_folder(study_id, filepaths, do_not_move=None):
     r"""Goes over the filepaths list and moves all the filepaths that are not
     used in any place to the upload folder of the study
 
@@ -950,6 +950,8 @@ def move_filepaths_to_upload_folder(study_id, filepaths):
         The study id to where the files should be returned to
     filepaths : list
         List of filepaths to move to the upload folder
+    do_not_move : list
+        List of filenames to delete rather than move to upload folder
     """
     with qdb.sql_connection.TRN:
         uploads_fp = join(get_mountpoint("uploads")[0][1], str(study_id))
@@ -965,12 +967,19 @@ def move_filepaths_to_upload_folder(study_id, filepaths):
 
             if x['fp_type'] in ('html_summary', 'html_summary_dir'):
                 _rm_files(qdb.sql_connection.TRN, x['fp'])
-            else:
-                destination = path_builder(basename(x['fp']))
+                continue
 
-                qdb.sql_connection.TRN.add_post_rollback_func(
-                    move, destination, x['fp'])
-                move(x['fp'], destination)
+            if do_not_move:
+                if basename(x['fp']) in do_not_move:
+                    _rm_files(qdb.sql_connection.TRN, x['fp'])
+                    continue
+
+            # if files were not removed, then they should be moved.
+            destination = path_builder(basename(x['fp']))
+            qdb.sql_connection.TRN.add_post_rollback_func(move,
+                                                          destination,
+                                                          x['fp'])
+            move(x['fp'], destination)
 
         qdb.sql_connection.TRN.execute()