Release candidate - April 09, 2018 (#2550)

* fix #1067 * addressing @ElDeveloper comments * fix #2512 * fix #1999 (#2514) * fix #1999 * fix template error * rm maintenance conditional * some improvements * fix #1053 * addressing @ElDeveloper comments * adding secret * WIP: EBI-ENA test (#2516) * fix Keemei (#2510) * rename preprocessed_data_id artifact-id * prints to debug * Keemei fix (#2517) * fix Keemei * delete keemei * keemei delete * rewrite_fastq=True * rmtree * add : * testing EBI * add secure * travis tests * @ElDeveloper key! * Fix typo * fix secure * rm single quote * Fix 2321 (#2522) * fix Keemei (#2510) * fix #2321 * fixing get studies * fix qiita_db * addressing @ElDeveloper comments * fixing error with skipIf * Illumina models 0318 (#2532) * fix Keemei (#2510) * updating illumina models * adding secure * Update index.rst (#2533) * fix Keemei (#2510) * Keemei fix (#2517) * fix Keemei * delete keemei * keemei delete * Update index.rst * Plugin API Doc (#2534) * fix Keemei (#2510) * plugin-api * improve format * Fix 1810 (#2523) * fix Keemei (#2510) * fix #2321 * fixing get studies * fix qiita_db * fix #1810 * adding self.ascp_pass * fix flake8 * flake8 * DOC: Cleanup of JavaScript libraries and licenses (#2536) * DOC: Cleanup of JavaScript libraries and licenses There were a few JavaScript files that were not used anywhere and we were missing a handful of license files. I've cleaned up all of that (and for consistency made all the "licence" into "license"). Fixes #2535 * DOC: Document all the JS files * BUG: Add natural sorting for data tables * adding initial download/redbiom info * adding docs * addressing @ElDeveloper comments * Fix aid str workflow (#2537) * fix Keemei (#2510) * fix int/str errors with workflows * adding a patch to fix all ints parameters * all -> any patch 64 * fix #2519 * fix #2524 * fix #2531 * fix #2529 * split qiita-cron-job into multiple commands * rm leftover docs * Update downloading.rst * fix_reference_order (#2547) * addressing @stephanieorch comments * addressing @ElDeveloper comments * Fix 2492 (#2544) * fix Keemei (#2510) * fix #2492 * flake8 * Redbiom -> redbiom @wasade * addressing @wasade comments * Update .travis.yml * Fix 2505 (#2545) * fix Keemei (#2510) * fix #2505 * fix errors * '' -> 'not provided' * Fix 2530 (#2546) * fix #2530 * fix errors * missed 1 error * more erros * fix-secure * improve available files display
qiita-spots · tanaes · May 2, 2018 · Feb 26, 2018 · Apr 6, 2018 · Apr 16, 2018
commit fd6e0ec25eb05d0a9d7745b09e88e2c6284c81f3
diff --git a/.travis.yml b/.travis.yml
@@ -4,6 +4,7 @@ sudo: false
 env:
   global:
     - PYTHON_VERSION=3.5
+    - secure: "OFyFddp26BbuaYxD2suftmbSfyaLcvleOL9heGh4IqqbvDVX+EcaqvmUALRaBL4HQlK5W0rO9H4pFsXuq26Xl3S2EKr+Pne4SDlw9ekmU3n6bWVNwReG7V+9c7OEgfNHbZ4pNwRzti62n1ESlRSZVkLVGg+pI0Ikvbh1+yUm9Eg="
   matrix:
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
@@ -19,9 +20,6 @@ before_install:
   - wget ftp://ftp.microbio.me/pub/qiita/ascp-install-3.5.4.102989-linux-64-qiita.sh -O ascp-install-3.5.4.102989-linux-64-qiita.sh
   - chmod +x ascp-install-3.5.4.102989-linux-64-qiita.sh
   - ./ascp-install-3.5.4.102989-linux-64-qiita.sh
-  # once we have ebi testing we should uncomment this line
-  # look for EBI below as it's part of this
-  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then openssl aes-256-cbc -K $encrypted_a2e23aea5f14_key -iv $encrypted_a2e23aea5f14_iv -in qiita_core/support_files/config_test_travis.cfg.enc -out qiita_core/support_files/config_test_travis.cfg -d ; fi
 install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
@@ -64,10 +62,6 @@ install:
   - source deactivate
   - source activate qiita
 before_script:
-  # EBI, see the end of before_install about why this block is commented out
-  # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then
-  #   export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
-  #   fi
   - qiita-env make --no-load-ontologies
   - qiita-test-install
 script:
@@ -79,8 +73,8 @@ script:
   - nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE
   - kill $QIITA_PID
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
-  - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi
-  - flake8 qiita_* setup.py scripts/*
+  - if [ ${TEST_ADD_STUDIES} == "True" ]; then all-qiita-cron-job; fi
+  - flake8 qiita_* setup.py scripts/qiita*
   - qiita pet webserver
 addons:
   postgresql: "9.3"

diff --git a/qiita_core/support_files/config_test_travis.cfg.enc b/qiita_core/support_files/config_test_travis.cfg.enc
diff --git a/qiita_db/archive.py b/qiita_db/archive.py
@@ -129,7 +129,7 @@ def get_merging_scheme_from_job(cls, job):
                     ppms = pms['parameters']
                     pparams = ','.join(
                         ['%s: %s' % (k, v) for k, v in ppp.values.items()
-                         if list(v)[0] != 'artifact' and k in ppms])
+                         if list(str(v))[0] != 'artifact' and k in ppms])
                     if pparams:
                         palgorithm = "%s (%s)" % (palgorithm, pparams)
             algorithm = '%s | %s' % (cname, palgorithm)

diff --git a/qiita_db/commands.py b/qiita_db/commands.py
@@ -186,45 +186,6 @@ def load_prep_template_from_cmd(prep_temp_path, study_id, data_type):
         prep_temp, qdb.study.Study(study_id), data_type)
 
 
-def load_parameters_from_cmd(name, fp, cmd_id):
-    """Add a new parameters entry on `table`
-
-    Parameters
-    ----------
-    fp : str
-        The filepath to the parameters file
-    cmd_id : int
-        The command to add the new default parameter set
-
-    Returns
-    -------
-    qiita_db.software.DefaultParameters
-        The newly parameter set object created
-
-    Raises
-    ------
-    ValueError
-        If the table does not exists on the DB
-        If the fp is not correctly formatted
-
-    Notes
-    -----
-    `fp` should be a tab-delimited text file following this format:
-        parameter_1<TAB>value
-        parameter_2<TAB>value
-        ...
-    """
-    cmd = qdb.software.Command(cmd_id)
-
-    try:
-        params = dict(tuple(l.strip().split('\t')) for l in open(fp, 'U'))
-    except ValueError:
-        raise ValueError("The format of the parameters files is not correct. "
-                         "The format is PARAMETER_NAME<tab>VALUE")
-
-    return qdb.software.DefaultParameters.create(name, cmd, **params)
-
-
 def update_artifact_from_cmd(filepaths, filepath_types, artifact_id):
     """Updates the artifact `artifact_id` with the given files
 

diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py
@@ -53,8 +53,8 @@ def _check_db_exists(db, conn_handler):
     return (db,) in dbs
 
 
-def create_layout_and_patch(test=False, verbose=False):
-    r"""Builds the SQL layout and applies all the patches
+def create_layout(test=False, verbose=False):
+    r"""Builds the SQL layout
 
     Parameters
     ----------
@@ -69,10 +69,6 @@ def create_layout_and_patch(test=False, verbose=False):
             qdb.sql_connection.TRN.add(f.read())
         qdb.sql_connection.TRN.execute()
 
-        if verbose:
-            print('Patching Database...')
-        patch(verbose=verbose, test=test)
-
 
 def _populate_test_db():
     with qdb.sql_connection.TRN:
@@ -191,30 +187,51 @@ def make_environment(load_ontologies, download_reference, add_demo_user):
 
     # Create the database
     print('Creating database')
+    create_settings_table = True
     admin_conn.autocommit = True
-    admin_conn.execute('CREATE DATABASE %s' % qiita_config.database)
+    try:
+        admin_conn.execute('CREATE DATABASE %s' % qiita_config.database)
+    except ValueError as error:
+        # if database exists ignore
+        msg = 'database "%s" already exists' % qiita_config.database
+        if msg in error.message:
+            print("Database exits, let's make sure it's test")
+            with qdb.sql_connection.TRN:
+                # Insert the settings values to the database
+                sql = """SELECT test FROM settings"""
+                qdb.sql_connection.TRN.add(sql)
+                is_test = qdb.sql_connection.TRN.execute_fetchlast()
+                if not is_test:
+                    print('Not a test database')
+                    raise
+                create_settings_table = False
+        else:
+            raise
     admin_conn.autocommit = False
 
     del admin_conn
     qdb.sql_connection.SQLConnectionHandler.close()
 
     with qdb.sql_connection.TRN:
         print('Inserting database metadata')
-        # Build the SQL layout into the database
-        with open(SETTINGS_FP, 'U') as f:
-            qdb.sql_connection.TRN.add(f.read())
-        qdb.sql_connection.TRN.execute()
+        test = qiita_config.test_environment
+        verbose = True
+        if create_settings_table:
+            # Build the SQL layout into the database
+            with open(SETTINGS_FP, 'U') as f:
+                qdb.sql_connection.TRN.add(f.read())
+            qdb.sql_connection.TRN.execute()
 
-        # Insert the settings values to the database
-        sql = """INSERT INTO settings (test, base_data_dir, base_work_dir)
-                 VALUES (%s, %s, %s)"""
-        qdb.sql_connection.TRN.add(
-            sql, [qiita_config.test_environment, qiita_config.base_data_dir,
-                  qiita_config.working_dir])
-        qdb.sql_connection.TRN.execute()
+            # Insert the settings values to the database
+            sql = """INSERT INTO settings (test, base_data_dir, base_work_dir)
+                     VALUES (%s, %s, %s)"""
+            qdb.sql_connection.TRN.add(
+                sql, [test, qiita_config.base_data_dir,
+                      qiita_config.working_dir])
+            qdb.sql_connection.TRN.execute()
+            create_layout(test=test, verbose=verbose)
 
-        create_layout_and_patch(test=qiita_config.test_environment,
-                                verbose=True)
+        patch(verbose=verbose, test=test)
 
         if load_ontologies:
             _add_ontology_data()
@@ -315,7 +332,8 @@ def drop_and_rebuild_tst_database():
         qdb.sql_connection.TRN.add(
             "UPDATE settings SET current_patch = 'unpatched'")
         # Create the database and apply patches
-        create_layout_and_patch(test=True)
+        create_layout(test=True)
+        patch(test=True)
 
         qdb.sql_connection.TRN.execute()
 

diff --git a/qiita_db/handlers/plugin.py b/qiita_db/handlers/plugin.py
@@ -93,6 +93,15 @@ def get(self, name, version):
 class CommandListHandler(OauthBaseHandler):
     @authenticate_oauth
     def post(self, name, version):
+        """Create new command for a plugin
+
+        Parameters
+        ----------
+        name : str
+            The name of the plugin
+        version : str
+            The version of the plugin
+        """
         with qdb.sql_connection.TRN:
             plugin = _get_plugin(name, version)
 

diff --git a/qiita_db/handlers/processing_job.py b/qiita_db/handlers/processing_job.py
@@ -101,7 +101,7 @@ def post(self, job_id):
 class ActiveStepHandler(OauthBaseHandler):
     @authenticate_oauth
     def post(self, job_id):
-        """Changes the current exectuion step of the given job
+        """Changes the current execution step of the given job
 
         Parameters
         ----------

diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py
@@ -969,7 +969,7 @@ def _update_children(self, mapping):
                 params, pending = qdb.sql_connection.TRN.execute_fetchflatten()
                 for pname, out_name in viewitems(pending[self.id]):
                     a_id = new_map[out_name]
-                    params[pname] = a_id
+                    params[pname] = str(a_id)
                     del pending[self.id]
                     # Link the input artifact with the child job
                     qdb.sql_connection.TRN.add(sql_link, [a_id, c.id])

diff --git a/qiita_db/study.py b/qiita_db/study.py
@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 from __future__ import division
+from collections import defaultdict
 from future.utils import viewitems
 from copy import deepcopy
 from itertools import chain
@@ -43,9 +44,6 @@
 import qiita_db as qdb
 
 
-_VALID_EBI_STATUS = ('not submitted', 'submitting', 'submitted')
-
-
 class Study(qdb.base.QiitaObject):
     r"""Study object to access to the Qiita Study information
 
@@ -81,8 +79,7 @@ class Study(qdb.base.QiitaObject):
     _table = "study"
     _portal_table = "study_portal"
     # The following columns are considered not part of the study info
-    _non_info = frozenset(["email", "study_title", "ebi_submission_status",
-                           "ebi_study_accession"])
+    _non_info = frozenset(["email", "study_title", "ebi_study_accession"])
 
     def _lock_non_sandbox(self):
         """Raises QiitaDBStatusError if study is non-sandboxed"""
@@ -212,10 +209,19 @@ def get_info(cls, study_ids=None, info_cols=None):
                 args.append(tuple(study_ids))
 
             qdb.sql_connection.TRN.add(sql, args)
-            res = qdb.sql_connection.TRN.execute_fetchindex()
-            if study_ids is not None and len(res) != len(study_ids):
+            rows = qdb.sql_connection.TRN.execute_fetchindex()
+            if study_ids is not None and len(rows) != len(study_ids):
                 raise qdb.exceptions.QiitaDBError(
                     'Non-portal-accessible studies asked for!')
+
+            res = []
+            for r in rows:
+                r = dict(r)
+                if 'ebi_study_accession' in info_cols:
+                    r['ebi_submission_status'] = cls(
+                        r['study_id']).ebi_submission_status
+                res.append(r)
+
             return res
 
     @classmethod
@@ -825,39 +831,58 @@ def ebi_submission_status(self):
         -------
         str
             The study EBI submission status
-        """
-        with qdb.sql_connection.TRN:
-            sql = """SELECT ebi_submission_status
-                     FROM qiita.{0}
-                     WHERE study_id = %s""".format(self._table)
-            qdb.sql_connection.TRN.add(sql, [self.id])
-            return qdb.sql_connection.TRN.execute_fetchlast()
 
-    @ebi_submission_status.setter
-    def ebi_submission_status(self, value):
-        """Sets the study's EBI submission status
-
-        Parameters
-        ----------
-        value : str {%s}
-            The new EBI submission status
-
-        Raises
-        ------
-        ValueError
-            If the status is not known
+        Notes
+        -----
+        There are 4 possible states: 'not submitted', 'submitting',
+        'submitted' & 'failed'. We are going to assume 'not submitted' if the
+        study doesn't have an accession, 'submitted' if it has an accession,
+        'submitting' if there are submit_to_EBI jobs running using the study
+        artifacts, & 'failed' if there are artifacts with failed jobs without
+        successful ones.
         """
-        if not (value in _VALID_EBI_STATUS or
-                value.startswith('failed')):
-            raise ValueError("Unknown status: %s" % value)
+        status = 'not submitted'
         with qdb.sql_connection.TRN:
-            sql = """UPDATE qiita.{}
-                     SET ebi_submission_status = %s
-                     WHERE study_id = %s""".format(self._table)
-            qdb.sql_connection.TRN.add(sql, [value, self.id])
-            qdb.sql_connection.TRN.execute()
-
-    ebi_submission_status.__doc__.format(', '.join(_VALID_EBI_STATUS))
+            if self.ebi_study_accession:
+                status = 'submitted'
+
+            plugin = qdb.software.Software.from_name_and_version(
+                'Qiita', 'alpha')
+            cmd = plugin.get_command('submit_to_EBI')
+
+            sql = """SELECT processing_job_id, command_parameters->>'artifact',
+                        processing_job_status
+                     FROM qiita.processing_job
+                     LEFT JOIN qiita.processing_job_status
+                        USING (processing_job_status_id)
+                     WHERE command_parameters->>'artifact' IN (
+                        SELECT artifact_id::text
+                        FROM qiita.study_artifact
+                        WHERE study_id = {0}) AND command_id = {1}""".format(
+                            self._id, cmd.id)
+            qdb.sql_connection.TRN.add(sql)
+            jobs = defaultdict(dict)
+            for info in qdb.sql_connection.TRN.execute_fetchindex():
+                jid, aid, js = info
+                jobs[js][aid] = jid
+
+            if 'queued' in jobs or 'running' in jobs:
+                status = 'submitting'
+            elif 'error' in jobs:
+                aids_error = []
+                aids_other = []
+                for s, aids in jobs.items():
+                    for aid in aids.keys():
+                        if s == 'error':
+                            aids_error.append(aid)
+                        else:
+                            aids_other.append(aid)
+                difference = set(aids_error) - set(aids_other)
+                if difference:
+                    status = ('Some artifact submissions failed: %s' %
+                              ', '.join(map(str, list(difference))))
+
+        return status
 
     @property
     def tags(self):

diff --git a/qiita_db/support_files/patches/64.sql b/qiita_db/support_files/patches/64.sql
@@ -0,0 +1,4 @@
+-- March 7, 2018
+-- delete ebi_submission_status from study
+
+ALTER TABLE qiita.study DROP ebi_submission_status;