WIP: Fixing EBI code

qiita_db/artifact.py

@@ -449,6 +449,31 @@ def can_be_submitted_to_ebi(self):
             qdb.sql_connection.TRN.add(sql, [self.id])
             return qdb.sql_connection.TRN.execute_fetchlast()
 
+    @property
+    def is_submitted_to_ebi(self):
+        """Whether the artifact has been submitted to EBI or not
+
+        Returns
+        -------
+        bool
+            True if the artifact has been submitted to EBI. False otherwise
+
+        raises
+        ------
+        QiitaDBOperationNotPermittedError
+            If the artifact cannot be submitted to EBI
+        """
+        with qdb.sql_connection.TRN:
+            if not self.can_be_submitted_to_ebi:
+                raise qdb.exceptions.QiitaDBOperationNotPermittedError(
+                    "Artifact %s cannot be submitted to EBI" % self.id)
+            sql = """SELECT EXISTS(
+                        SELECT *
+                        FROM qiita.ebi_run_accession
+                        WHERE artifact_id = %s)"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
     @property
     def ebi_run_accessions(self):
         """The EBI run accessions attached to this artifact

qiita_db/test/test_artifact.py

@@ -327,6 +327,16 @@ def test_can_be_submitted_to_ebi(self):
         self.assertTrue(qdb.artifact.Artifact(3).can_be_submitted_to_ebi)
         self.assertFalse(qdb.artifact.Artifact(4).can_be_submitted_to_ebi)
 
+    def test_is_submitted_to_ebi(self):
+        with self.assertRaises(
+                qdb.exceptions.QiitaDBOperationNotPermittedError):
+            qdb.artifact.Artifact(1).is_submitted_to_ebi
+        self.assertTrue(qdb.artifact.Artifact(2).is_submitted_to_ebi)
+        self.assertFalse(qdb.artifact.Artifact(3).is_submitted_to_ebi)
+        with self.assertRaises(
+                qdb.exceptions.QiitaDBOperationNotPermittedError):
+            qdb.artifact.Artifact(4).is_submitted_to_ebi
+
     def test_ebi_run_accessions(self):
         exp = {'1.SKB1.640202': 'ERR0000001',
                '1.SKB2.640194': 'ERR0000002',

qiita_pet/handlers/compute.py

@@ -13,7 +13,7 @@
 from qiita_db.study import Study
 from qiita_db.exceptions import QiitaDBUnknownIDError
 from qiita_db.util import get_mountpoint
-from qiita_db.metadata_template import PrepTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
 
 from os.path import join, exists
 

qiita_pet/handlers/preprocessing_handlers.py

@@ -5,7 +5,7 @@
 from qiita_db.data import RawData
 from qiita_db.parameters import (PreprocessedIlluminaParams,
                                  Preprocessed454Params)
-from qiita_db.metadata_template import PrepTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
 from qiita_ware.context import submit
 from qiita_core.util import execute_as_transaction
 

qiita_pet/handlers/study_handlers/description_handlers.py

@@ -22,10 +22,11 @@
 from qiita_db.study import Study
 from qiita_db.data import RawData, PreprocessedData, ProcessedData
 from qiita_db.ontology import Ontology
-from qiita_db.metadata_template import (PrepTemplate, SampleTemplate,
-                                        load_template_to_dataframe,
-                                        SAMPLE_TEMPLATE_COLUMNS,
-                                        looks_like_qiime_mapping_file)
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
+from qiita_db.metadata_template.util import (load_template_to_dataframe,
+                                             looks_like_qiime_mapping_file)
+from qiita_db.metadata_template.constants import SAMPLE_TEMPLATE_COLUMNS
 from qiita_db.util import convert_to_id, get_mountpoint
 from qiita_db.exceptions import (QiitaDBUnknownIDError, QiitaDBColumnError,
                                  QiitaDBExecutionError, QiitaDBDuplicateError,

qiita_pet/handlers/study_handlers/ebi_handlers.py

@@ -13,9 +13,10 @@
 from qiita_ware.demux import stats as demux_stats
 from qiita_ware.dispatchable import submit_to_ebi
 from qiita_db.data import PreprocessedData
-from qiita_db.metadata_template import (PrepTemplate, SampleTemplate,
-                                        SAMPLE_TEMPLATE_COLUMNS,
-                                        PREP_TEMPLATE_COLUMNS)
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
+from qiita_db.metadata_template.constants import (SAMPLE_TEMPLATE_COLUMNS,
+                                                  PREP_TEMPLATE_COLUMNS)
 from qiita_db.study import Study
 from qiita_db.exceptions import QiitaDBUnknownIDError
 from qiita_pet.handlers.base_handlers import BaseHandler

qiita_pet/handlers/study_handlers/listing_handlers.py

@@ -17,7 +17,7 @@
 from qiita_db.user import User
 from qiita_db.study import Study, StudyPerson
 from qiita_db.search import QiitaStudySearch
-from qiita_db.metadata_template import SampleTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
 from qiita_db.logger import LogEntry
 from qiita_db.exceptions import QiitaDBIncompatibleDatatypeError
 from qiita_db.util import get_table_cols

qiita_pet/handlers/study_handlers/metadata_summary_handlers.py

@@ -11,7 +11,8 @@
 
 from qiita_ware.util import stats_from_df
 from qiita_db.study import Study
-from qiita_db.metadata_template import SampleTemplate, PrepTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
 from qiita_db.exceptions import QiitaDBUnknownIDError
 from qiita_pet.handlers.base_handlers import BaseHandler
 from qiita_core.util import execute_as_transaction

qiita_pet/handlers/study_handlers/vamps_handlers.py

@@ -13,7 +13,8 @@
 from qiita_ware.demux import stats as demux_stats
 from qiita_ware.dispatchable import submit_to_VAMPS
 from qiita_db.data import PreprocessedData
-from qiita_db.metadata_template import PrepTemplate, SampleTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
 from qiita_db.study import Study
 from qiita_db.exceptions import QiitaDBUnknownIDError
 from qiita_pet.handlers.base_handlers import BaseHandler

qiita_pet/uimodules/prep_template_tab.py

@@ -12,14 +12,15 @@
 
 from future.utils import viewitems
 
-from qiita_db.util import (get_filetypes, get_files_from_uploads_folders,
+from qiita_db.util import (get_artifact_types, get_files_from_uploads_folders,
                            get_data_types, convert_to_id, get_filepath_types)
 from qiita_pet.util import convert_text_html
 from qiita_db.study import Study
 from qiita_db.data import RawData
 from qiita_db.ontology import Ontology
-from qiita_db.metadata_template import (PrepTemplate, TARGET_GENE_DATA_TYPES,
-                                        PREP_TEMPLATE_COLUMNS_TARGET_GENE)
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.constants import (
+    TARGET_GENE_DATA_TYPES, PREP_TEMPLATE_COLUMNS_TARGET_GENE)
 from qiita_db.parameters import (Preprocessed454Params,
                                  PreprocessedIlluminaParams)
 from qiita_pet.util import STATUS_STYLER, is_localhost, EBI_LINKIFIER
@@ -153,7 +154,7 @@ def render(self, study, prep_template, full_access, ena_terms,
 
         filetypes = sorted(
             ((ft, ft_id, fp_type_by_ft[ft])
-             for ft, ft_id in viewitems(get_filetypes())),
+             for ft, ft_id in viewitems(get_artifact_types())),
             key=itemgetter(1))
         files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
 

qiita_pet/uimodules/preprocessed_data_tab.py

@@ -9,7 +9,7 @@
 from qiita_core.util import execute_as_transaction
 from qiita_db.study import Study
 from qiita_db.data import PreprocessedData
-from qiita_db.metadata_template import PrepTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
 from qiita_db.ontology import Ontology
 from qiita_db.util import convert_to_id
 from qiita_pet.util import convert_text_html

qiita_pet/uimodules/study_information_tab.py

@@ -14,7 +14,7 @@
 from qiita_core.util import execute_as_transaction
 from qiita_db.util import get_files_from_uploads_folders, get_data_types
 from qiita_db.study import StudyPerson
-from qiita_db.metadata_template import SampleTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
 from qiita_pet.util import linkify, is_localhost, EBI_LINKIFIER
 from .base_uimodule import BaseUIModule
 

qiita_ware/commands.py

@@ -16,7 +16,8 @@
 
 from qiita_db.study import Study
 from qiita_db.data import PreprocessedData
-from qiita_db.metadata_template import PrepTemplate, SampleTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.sample_template import SampleTemplate
 from qiita_db.logger import LogEntry
 from qiita_core.qiita_settings import qiita_config
 from qiita_ware.ebi import EBISubmission

qiita_ware/dispatchable.py

@@ -13,7 +13,7 @@
 from qiita_ware.commands import submit_EBI, submit_VAMPS
 from qiita_db.study import Study
 from qiita_db.analysis import Analysis
-from qiita_db.metadata_template import PrepTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
 from qiita_db.data import RawData, PreprocessedData
 
 

qiita_ware/ebi.py

@@ -18,9 +18,7 @@
 from qiita_db.logger import LogEntry
 from qiita_db.ontology import Ontology
 from qiita_db.util import convert_to_id, get_mountpoint
-from qiita_db.study import Study
-from qiita_db.data import PreprocessedData
-from qiita_db.metadata_template import PrepTemplate, SampleTemplate
+from qiita_db.artifact import Artifact
 
 
 def clean_whitespace(text):
@@ -42,7 +40,7 @@ def clean_whitespace(text):
 class EBISubmission(object):
     """Define an EBI submission, generate submission files and submit
 
-    Submit a preprocessed data to EBI
+    Submit an artifact to EBI
 
     The steps for EBI submission are:
     1. Validate that we have all required info to submit
@@ -53,8 +51,8 @@ class EBISubmission(object):
 
     Parameters
     ----------
-    preprocessed_data_id : int
-        The preprocesssed data id to submit
+    artifact_id : int
+        The artifact id to submit
     action : str
         The action to perform. Valid options see
         EBISubmission.valid_ebi_actions
@@ -63,9 +61,10 @@ class EBISubmission(object):
     ------
     EBISubmissionError
         - If the action is not in EBISubmission.valid_ebi_actions
-        - If the preprocessed data has been already submitted to EBI
-        - If the status of the study attached to the preprocessed data is
-        submitting
+        - If the artifact cannot be submitted to EBI
+        - If the artifact has been already submitted to EBI and the action
+        is different from 'MODIFY'
+        - If the status of the study attached to the artifact is `submitting`
         - If the prep template investigation type is not in the
         ena_ontology.terms or not in the ena_ontology.user_defined_terms
         - If the submission is missing required EBI fields either in the sample
@@ -93,7 +92,7 @@ class EBISubmission(object):
     xsi_noNSL = "ftp://ftp.sra.ebi.ac.uk/meta/xsd/sra_1_3/SRA.%s.xsd"
     experiment_library_fields = ['library_strategy']
 
-    def __init__(self, preprocessed_data_id, action):
+    def __init__(self, artifact_id, action):
         error_msgs = []
 
         if action not in self.valid_ebi_actions:
@@ -105,14 +104,20 @@ def __init__(self, preprocessed_data_id, action):
 
         ena_ontology = Ontology(convert_to_id('ENA', 'ontology'))
         self.action = action
-        self.preprocessed_data = PreprocessedData(preprocessed_data_id)
-        self.study = Study(self.preprocessed_data.study)
-        self.sample_template = SampleTemplate(self.study.sample_template)
-        self.prep_template = PrepTemplate(self.preprocessed_data.prep_template)
-
-        if self.preprocessed_data.is_submitted_to_ebi and action != 'MODIFY':
-            error_msg = ("Cannot resubmit! Preprocessed data %d has already "
-                         "been submitted to EBI.")
+        self.artifact = Artifact(artifact_id)
+        if not self.artifact.can_be_submitted_to_ebi:
+            error_msg = ("Artifact %d cannot be submitted to EBI"
+                         % self.artifact.id)
+            LogEntry.create('Runtime', error_msg)
+            raise EBISubmissionError(error_msg)
+
+        self.study = self.artifact.study
+        self.sample_template = self.study.sample_template
+        self.prep_template = self.artifact.prep_templates[0]
+
+        if self.artifact.is_submitted_to_ebi and action != 'MODIFY':
+            error_msg = ("Cannot resubmit! Artifact %d has already "
+                         "been submitted to EBI." % artifact_id)
             LogEntry.create('Runtime', error_msg)
             raise EBISubmissionError(error_msg)
 
@@ -123,7 +128,7 @@ def __init__(self, preprocessed_data_id, action):
             LogEntry.create('Runtime', error_msg)
             raise EBISubmissionError(error_msg)
 
-        self.preprocessed_data_id = preprocessed_data_id
+        self.artifact_id = artifact_id
         self.study_title = self.study.title
         self.study_abstract = self.study.info['study_abstract']
 
@@ -141,7 +146,7 @@ def __init__(self, preprocessed_data_id, action):
                               "one of the user-defined terms in the ENA "
                               "ontology." % it)
         _, base_fp = get_mountpoint("preprocessed_data")[0]
-        self.ebi_dir = '%d_ebi_submission' % preprocessed_data_id
+        self.ebi_dir = '%d_ebi_submission' % artifact_id
         self.full_ebi_dir = join(base_fp, self.ebi_dir)
         self.ascp_reply = join(self.full_ebi_dir, 'ascp_reply.txt')
         self.curl_reply = join(self.full_ebi_dir, 'curl_reply.xml')
@@ -208,8 +213,8 @@ def __init__(self, preprocessed_data_id, action):
                               "model: %s" % (', '.join(nvim)))
         if error_msgs:
             error_msgs = ("Errors found during EBI submission for study #%d, "
-                          "preprocessed data #%d and prep template #%d:\n%s"
-                          % (self.study.id, preprocessed_data_id,
+                          "artifact #%d and prep template #%d:\n%s"
+                          % (self.study.id, artifact_id,
                              self.prep_template.id, '\n'.join(error_msgs)))
             LogEntry.create('Runtime', error_msgs)
             raise EBISubmissionError(error_msgs)
@@ -224,21 +229,21 @@ def __init__(self, preprocessed_data_id, action):
             self.prep_template.ebi_experiment_accessions
 
     def _get_study_alias(self):
-        """Format alias using ``self.preprocessed_data_id``"""
+        """Format alias using ``self.study_id``"""
         study_alias_format = '%s_sid_%s'
         return study_alias_format % (
             qiita_config.ebi_organization_prefix,
             escape(clean_whitespace(str(self.study.id))))
 
     def _get_sample_alias(self, sample_name):
-        """Format alias using ``self.preprocessed_data_id``, `sample_name`"""
+        """Format alias using ``self.study_id``, `sample_name`"""
         alias = "%s:%s" % (self._get_study_alias(),
                            escape(clean_whitespace(str(sample_name))))
         self._sample_aliases[alias] = sample_name
         return alias
 
     def _get_experiment_alias(self, sample_name):
-        """Format alias using ``self.preprocessed_data_id``, and `sample_name`
+        """Format alias using ``self.prep_template.id``, and `sample_name`
 
         Currently, this is identical to _get_sample_alias above, since we are
         only going to allow submission of one prep for each sample
@@ -252,19 +257,19 @@ def _get_experiment_alias(self, sample_name):
         return alias
 
     def _get_submission_alias(self):
-        """Format alias using ``self.preprocessed_data_id``"""
-        safe_preprocessed_data_id = escape(
-            clean_whitespace(str(self.preprocessed_data_id)))
+        """Format alias using ``self.artifact_id``"""
+        safe_artifact_id = escape(
+            clean_whitespace(str(self.artifact_id)))
         submission_alias_format = '%s_submission_%s'
         return submission_alias_format % (qiita_config.ebi_organization_prefix,
-                                          safe_preprocessed_data_id)
+                                          safe_artifact_id)
 
     def _get_run_alias(self, sample_name):
         """Format alias using `sample_name`
         """
         alias = '%s_ppdid_%s:%s' % (
             qiita_config.ebi_organization_prefix,
-            escape(clean_whitespace(str(self.preprocessed_data_id))),
+            escape(clean_whitespace(str(self.artifact_id))),
             sample_name)
         self._run_aliases[alias] = sample_name
         return alias
@@ -922,13 +927,13 @@ def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None):
             - The demux file couldn't be read
             - All samples are removed
         """
-        ppd = self.preprocessed_data
+        ar = self.artifact
 
         dir_not_exists = not isdir(self.full_ebi_dir)
         if dir_not_exists or rewrite_fastq:
             makedirs(self.full_ebi_dir)
 
-            demux = [path for _, path, ftype in ppd.get_filepaths()
+            demux = [path for _, path, ftype in ar.get_filepaths()
                      if ftype == 'preprocessed_demux'][0]
 
             demux_samples = set()

qiita_ware/metadata_pipeline.py

@@ -8,12 +8,12 @@
 
 from future.utils import viewvalues, viewkeys
 
-from qiita_db.metadata_template import (load_template_to_dataframe,
-                                        SampleTemplate, PrepTemplate,
-                                        PREP_TEMPLATE_COLUMNS,
-                                        PREP_TEMPLATE_COLUMNS_TARGET_GENE,
-                                        CONTROLLED_COLS,
-                                        TARGET_GENE_DATA_TYPES)
+from qiita_db.metadata_template.util import load_template_to_dataframe
+from qiita_db.metadata_template.sample_template import SampleTemplate
+from qiita_db.metadata_template.prep_template import PrepTemplate
+from qiita_db.metadata_template.constants import (
+    PREP_TEMPLATE_COLUMNS, PREP_TEMPLATE_COLUMNS_TARGET_GENE, CONTROLLED_COLS,
+    TARGET_GENE_DATA_TYPES)
 from qiita_db.util import convert_from_id
 from qiita_ware.exceptions import QiitaWareError
 

qiita_ware/processing_pipeline.py

@@ -16,7 +16,7 @@
 from qiita_ware.wrapper import ParallelWrapper
 from qiita_db.logger import LogEntry
 from qiita_db.data import RawData
-from qiita_db.metadata_template import TARGET_GENE_DATA_TYPES
+from qiita_db.metadata_template.constants import TARGET_GENE_DATA_TYPES
 from qiita_db.reference import Reference