Actually testing plugins in travis

.travis.yml

@@ -26,12 +26,18 @@ install:
     pyzmq networkx pyparsing natsort mock future libgfortran seaborn
     'pandas>=0.15' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate env_name
-  - pip install sphinx sphinx-bootstrap-theme coveralls ipython[all]==2.4.1
+  - pip install sphinx sphinx-bootstrap-theme coveralls ipython[all]==2.4.1 HTTPretty requests
+  - pushd qiita_plugins/qiita_client
+  - travis_retry pip install .
+  - popd
   - travis_retry pip install .
   - pushd qiita_plugins/target_gene
   - travis_retry pip install .
   - popd
-  - pushd qiita_plugins/qiita_client
+  - pushd qiita_plugins/biom_type
+  - travis_retry pip install .
+  - popd
+  - pushd qiita_plugins/target_gene_type
   - travis_retry pip install .
   - popd
 script:
@@ -45,7 +51,8 @@ script:
   - qiita-env make --no-load-ontologies
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
   - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v; fi
+  # - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v qiita_plugins/* qiita_*; fi
+  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v qiita_plugins/*; fi
   - flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install
   - ls -R /home/travis/miniconda3/envs/env_name/lib/python2.7/site-packages/qiita_pet/support_files/doc/
   - qiita pet webserver

qiita_core/configuration_manager.py

@@ -105,6 +105,10 @@ class ConfigurationManager(object):
         The VAMPS URL
     conf_fp : str
         The filepath for the configuration file that is loaded
+    portal : str
+        The portal under the Qiita instance is running under
+    portal_dir : str
+        The portal subdirectory used in the URL
     portal_fp : str
         The filepath to the portal styling config file
     plugin_launcher : str
@@ -161,7 +165,6 @@ def _get_main(self, config):
                              self.working_dir)
         self.max_upload_size = config.getint('main', 'MAX_UPLOAD_SIZE')
         self.require_approval = config.getboolean('main', 'REQUIRE_APPROVAL')
-        self.portal = config.get('main', 'PORTAL')
         self.plugin_launcher = config.get('main', 'PLUGIN_LAUNCHER')
 
         self.valid_upload_extension = [ve.strip() for ve in config.get(
@@ -240,3 +243,12 @@ def _get_vamps(self, config):
 
     def _get_portal(self, config):
         self.portal_fp = config.get('portal', 'PORTAL_FP')
+        self.portal = config.get('portal', 'PORTAL')
+        self.portal_dir = config.get('portal', 'PORTAL_DIR')
+        if self.portal_dir:
+            if not self.portal_dir.startswith('/'):
+                self.portal_dir = "/%s" % self.portal_dir
+            if self.portal_dir.endswith('/'):
+                self.portal_dir = self.portal_dir[:-1]
+        else:
+            self.portal_dir = ""

qiita_core/support_files/config_test.cfg

@@ -39,9 +39,6 @@ BASE_DATA_DIR =
 # Valid upload extension, comma separated. Empty for no uploads
 VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual
 
-# Portal the site is working under
-PORTAL = QIITA
-
 # Script used for launching plugins
 PLUGIN_LAUNCHER = qiita-plugin-launcher
 
@@ -144,5 +141,12 @@ URL = https://vamps.mbl.edu/mobe_workshop/getfile.php
 
 # ----------------------------- Portal settings -----------------------------
 [portal]
+
+# Portal the site is working under
+PORTAL = QIITA
+
+# Portal subdirectory
+PORTAL_DIR =
+
 # Full path to portal styling config file
 PORTAL_FP =

qiita_db/analysis.py

@@ -683,13 +683,11 @@ def share(self, user):
         user: User object
             The user to share the analysis with
         """
-        with qdb.sql_connection.TRN:
-            self._lock_check()
-
-            # Make sure the analysis is not already shared with the given user
-            if user.id in self.shared_with:
-                return
+        # Make sure the analysis is not already shared with the given user
+        if user.id == self.owner or user.id in self.shared_with:
+            return
 
+        with qdb.sql_connection.TRN:
             sql = """INSERT INTO qiita.analysis_users (analysis_id, email)
                      VALUES (%s, %s)"""
             qdb.sql_connection.TRN.add(sql, [self._id, user.id])
@@ -704,8 +702,6 @@ def unshare(self, user):
             The user to unshare the analysis with
         """
         with qdb.sql_connection.TRN:
-            self._lock_check()
-
             sql = """DELETE FROM qiita.analysis_users
                      WHERE analysis_id = %s AND email = %s"""
             qdb.sql_connection.TRN.add(sql, [self._id, user.id])
@@ -933,9 +929,7 @@ def _build_biom_tables(self, grouped_samples, rarefaction_depth=None,
                     biom_table.filter(selected_samples, axis='sample',
                                       inplace=True)
                     if len(biom_table.ids()) == 0:
-                        raise RuntimeError(
-                            "All samples filtered out from Artifact %s due "
-                            "to selected samples" % aid)
+                        continue
 
                     if rename_dup_samples:
                         ids_map = {_id: "%d.%s" % (aid, _id)
@@ -947,6 +941,12 @@ def _build_biom_tables(self, grouped_samples, rarefaction_depth=None,
                     else:
                         new_table = new_table.merge(biom_table)
 
+                if not new_table or len(new_table.ids()) == 0:
+                    # if we get to this point the only reason for failure is
+                    # rarefaction
+                    raise RuntimeError("All samples filtered out from "
+                                       "analysis due to rarefaction level")
+
                 # add the metadata column for study the samples come from,
                 # this is useful in case the user download the bioms
                 study_md = {'study': artifact.study.title,
@@ -977,27 +977,15 @@ def _build_mapping_file(self, samples, rename_dup_samples=False):
         """Builds the combined mapping file for all samples
            Code modified slightly from qiime.util.MetadataMap.__add__"""
         with qdb.sql_connection.TRN:
-            # query to get the latest qiime mapping file
-            sql = """SELECT filepath
-                     FROM qiita.filepath
-                        JOIN qiita.prep_template_filepath USING (filepath_id)
-                        JOIN qiita.prep_template USING (prep_template_id)
-                        JOIN qiita.filepath_type USING (filepath_type_id)
-                     WHERE filepath_type = 'qiime_map'
-                        AND artifact_id IN (SELECT *
-                                            FROM qiita.find_artifact_roots(%s))
-                     ORDER BY filepath_id DESC LIMIT 1"""
-            _, fp = qdb.util.get_mountpoint('templates')[0]
-
             all_ids = set()
             to_concat = []
             for aid, samps in viewitems(samples):
-                qdb.sql_connection.TRN.add(sql, [aid])
-                qm_fp = qdb.sql_connection.TRN.execute_fetchindex()[0][0]
+                qiime_map_fp = qdb.artifact.Artifact(
+                    aid).prep_templates[0].qiime_map_fp
 
                 # Parse the mapping file
                 qm = qdb.metadata_template.util.load_template_to_dataframe(
-                    join(fp, qm_fp), index='#SampleID')
+                    qiime_map_fp, index='#SampleID')
 
                 # if we are not going to merge the duplicated samples
                 # append the aid to the sample name
@@ -1011,6 +999,16 @@ def _build_mapping_file(self, samples, rename_dup_samples=False):
                     samps = set(samps) - all_ids
                     all_ids.update(samps)
 
+                # appending study metadata to the analysis
+                study = qdb.artifact.Artifact(aid).study
+                study_owner = study.owner
+                study_info = study.info
+                pi = study_info['principal_investigator']
+                qm['qiita_study_title'] = study.title
+                qm['qiita_study_alias'] = study.info['study_alias']
+                qm['qiita_owner'] = study_owner.info['name']
+                qm['qiita_principal_investigator'] = pi.name
+
                 qm = qm.loc[samps]
                 to_concat.append(qm)
 
@@ -1029,7 +1027,7 @@ def _build_mapping_file(self, samples, rename_dup_samples=False):
             _, base_fp = qdb.util.get_mountpoint(self._table)[0]
             mapping_fp = join(base_fp, "%d_analysis_mapping.txt" % self._id)
             merged_map.to_csv(mapping_fp, index_label='#SampleID',
-                              na_rep='unknown', sep='\t')
+                              na_rep='unknown', sep='\t', encoding='utf-8')
 
             self._add_file("%d_analysis_mapping.txt" % self._id, "plain_text")
 

qiita_db/metadata_template/constants.py

@@ -53,17 +53,14 @@
     'demultiplex': Restriction(
         columns={'barcode': 'varchar',
                  'primer': 'varchar'},
-        error_msg="Demultiplexing disabled. You will not be able to "
-                  "preprocess your raw data"),
+        error_msg="Demultiplexing disabled."),
     # The following columns are required by Qiita to know how to execute split
     # libraries using QIIME over a study with multiple illumina lanes
     'demultiplex_multiple': Restriction(
         columns={'barcode': 'varchar',
                  'primer': 'varchar',
                  'run_prefix': 'varchar'},
-        error_msg="Demultiplexing with multiple input files disabled. If your "
-                  "raw data includes multiple raw input files, you will not "
-                  "be able to preprocess your raw data")
+        error_msg="Demultiplexing with multiple input files disabled.")
 }
 
 # This list is useful to have if we want to loop through all the restrictions

qiita_db/processing_job.py

@@ -306,9 +306,12 @@ def _generate_cmd(self):
         software = self.command.software
         plugin_start_script = software.start_script
         plugin_env_script = software.environment_script
+        # Appending the portal URL so the job requests the information from the
+        # portal server that submitted the job
+        url = "%s%s" % (qiita_config.base_url, qiita_config.portal_dir)
         cmd = '%s "%s" "%s" "%s" "%s" "%s"' % (
             qiita_config.plugin_launcher, plugin_env_script,
-            plugin_start_script, qiita_config.base_url, self.id, job_dir)
+            plugin_start_script, url, self.id, job_dir)
         return cmd
 
     def submit(self):
@@ -324,7 +327,7 @@ def submit(self):
             raise qdb.exceptions.QiitaDBOperationNotPermittedError(
                 "Can't submit job, not in 'in_construction' or "
                 "'waiting' status. Current status: %s" % status)
-        self._set_status = 'queued'
+        self._set_status('queued')
         cmd = self._generate_cmd()
         p = Process(target=_job_submitter, args=(self, cmd))
         p.start()

qiita_db/support_files/populate_test_db.sql

@@ -427,7 +427,7 @@ INSERT INTO qiita.job (data_type_id, job_status_id, command_id, options, input_f
 INSERT INTO qiita.job_results_filepath (job_id, filepath_id) VALUES (1, 13), (2, 14);
 
 -- Insert Analysis
-INSERT INTO qiita.analysis (email, name, description, analysis_status_id, pmid) VALUES ('test@foo.bar', 'SomeAnalysis', 'A test analysis', 1, '121112'), ('test@foo.bar', 'SomeSecondAnalysis', 'Another test analysis', 1, '22221112');
+INSERT INTO qiita.analysis (email, name, description, analysis_status_id, pmid) VALUES ('test@foo.bar', 'SomeAnalysis', 'A test analysis', 1, '121112'), ('admin@foo.bar', 'SomeSecondAnalysis', 'Another test analysis', 1, '22221112');
 INSERT INTO qiita.analysis_portal (analysis_id, portal_type_id) VALUES (1, 1), (2, 1);
 -- Insert Analysis Workflow
 INSERT INTO qiita.analysis_workflow (analysis_id, step) VALUES (1, 3), (2, 3);