Release candidate (#2298)

.gitignore

@@ -58,6 +58,5 @@ gg_13_8-*
 # sphinx documentation
 qiita_pet/static/doc/
 
-# moi static files
-qiita_pet/static/vendor/js/moi.js
-qiita_pet/static/vendor/js/moi_list.js
+# webdis log
+webdis.log

.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: python
 sudo: false
 env:
@@ -6,10 +7,10 @@ env:
   matrix:
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_db
     - TEST_ADD_STUDIES=False COVER_PACKAGE=qiita_pet
-    - TEST_ADD_STUDIES=False COVER_PACKAGE="qiita_core qiita_ware"
-    - TEST_ADD_STUDIES=True
+    - TEST_ADD_STUDIES=True COVER_PACKAGE="qiita_core qiita_ware"
 before_install:
   - redis-server --version
+  - redis-server --port 7777 &
   - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - chmod +x miniconda.sh
   - ./miniconda.sh -b
@@ -27,43 +28,54 @@ install:
   # install a few of the dependencies that pip would otherwise try to install
   # when intalling scikit-bio
   - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8
-    pyzmq networkx pyparsing natsort mock future libgfortran seaborn
+    pyzmq networkx pyparsing natsort mock future libgfortran seaborn nltk
     'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1'
   - source activate qiita
   - pip install -U pip
-  - pip install sphinx sphinx-bootstrap-theme 'ipython[all]==2.4.1' nose-timer codecov
-  - travis_retry pip install . --process-dependency-links
+  - pip install sphinx sphinx-bootstrap-theme nose-timer codecov
   - 'echo "backend: Agg" > matplotlibrc'
+  - git clone https://github.com/nicolasff/webdis
+  - pushd webdis
+  - make
+  - ./webdis &
+  - popd
+  - travis_retry pip install . --process-dependency-links
   # Install the biom plugin so we can run the analysis tests
   - pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
   - pip install https://github.com/qiita-spots/qtp-biom/archive/master.zip --process-dependency-links
   - export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+  # loading redbiom with Qiita's test set
+  # but first let's make sure redis is empty
+  # following https://github.com/biocore/redbiom/blob/master/Makefile
+  - export REDBIOM_HOST=http://127.0.0.1:7379
+  - curl -s http://127.0.0.1:7379/FLUSHALL > /dev/null
+  - redbiom admin scripts-writable
+  - redbiom admin create-context --name "qiita-test" --description "qiita-test context"
+  - redbiom admin load-sample-metadata --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+  - redbiom admin load-sample-metadata-search --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+  - redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test --tag 4
+  - redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table-for_redbiom_tests.biom --context qiita-test --tag 5
   - mkdir ~/.qiita_plugins
+  - export REDBIOM_HOST=http://127.0.0.1:7379
   - cp $PWD/qiita_core/support_files/BIOM\ type_2.1.4.conf ~/.qiita_plugins
 before_script:
-    # Some of the tests rely on the plugin system to complete successfuly.
-    # Thus, we need a qiita webserver running to be able to execute the tests.
-  - export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
   # EBI, see the end of before_install about why this block is commented out
   # - if [ ${TRAVIS_PULL_REQUEST} == "false" ]; then
   #   export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
-  #   export MOI_CONFIG_FP=`pwd`/qiita_core/support_files/config_test_travis.cfg;
   #   fi
-  - ipython profile create qiita-general --parallel
-  - qiita-env start_cluster qiita-general
   - qiita-env make --no-load-ontologies
-  - |
-    if [ ${TEST_ADD_STUDIES} == "False" ]; then
-        qiita pet webserver --no-build-docs start &
-    fi
+  - qiita-test-install
 script:
+  # Some of the tests rely on the plugin system to complete successfuly.
+  # Thus, we need a qiita webserver running to be able to execute the tests.
+  - qiita pet webserver --no-build-docs start &
   - sleep 5
+  - QIITA_PID=$!
+  - nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE
+  - kill $QIITA_PID
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi
   - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi
-  - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE; fi
   - flake8 qiita_* setup.py scripts/*
-  - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/
   - qiita pet webserver
 addons:
   postgresql: "9.3"

CONTRIBUTING.md

@@ -1,10 +1,10 @@
-#Contributing to Qiita
+# Contributing to Qiita
 
 Qiita is an open source software package, and we welcome community contributions. You can find the source code and test code for Qiita under public revision control in the Qiita git repository on [GitHub](http://github.com/biocore/qiita). We very much welcome contributions.
 
 This document covers what you should do to get started with contributing to Qiita. You should read this whole document before considering submitting code to Qiita. This will save time for both you and the Qiita developers.
 
-#General Notes on Development
+# General Notes on Development
 
 Adding source code to Qiita, can take place in three different modules:
 
@@ -17,7 +17,15 @@ always take into consideration how these new features affect users and whether
 or not adding a new section or document to the documentation (found under the
 `doc` folder) would be useful.
 
-###The Qiita development rules
+### Repository branch structure
+
+The Qiita repository contains three branches:
+
+* `master`: This branch reflects the code deployed in our [main Qiita server](http://qiita.microbio.me).
+* `dev`: This branch is the active development branch. All new Pull Requests should be performed against this branch.
+* `release-candidate`: This branch is used to freeze the code from the `dev` branch, so we can deploy in our test servers and exercise the code extensively before deploying in our main system. Code freezes typically occur one week before the scheduled deployment. Check our [milestones page](https://github.com/biocore/qiita/milestones) to see the scheduled deployments.
+
+### The Qiita development rules
 
 Since Qiita is a package that is continuously growing, we found ourselves in a position where development rules needed to be established so we can reduce both development and reviewer time. These rules are:
 
@@ -46,49 +54,47 @@ Since Qiita is a package that is continuously growing, we found ourselves in a p
 9. Last but not least, you are working as part of a team and you should try to help others when possible.
 
 
-###Configuration file
+### Configuration file
 
-The Qiita configuration file determines how the package interacts with your system’s resources (redis, postgres and the IPython cluster). Thus you should review the documentation detailed [here](https://docs.google.com/document/d/1u7kwLP31NM513-8xwpwvLbSQxYu0ehI6Jau1APR13e0/edit#), but especially bear in mind the following points:
+The Qiita configuration file determines how the package interacts with your system’s resources (redis and postgres). Thus you should review the documentation detailed [here](https://docs.google.com/document/d/1u7kwLP31NM513-8xwpwvLbSQxYu0ehI6Jau1APR13e0/edit#), but especially bear in mind the following points:
 
 * An example version of this file can be found here `qiita_core/support_files/qiita_config.txt` and if you don’t set a `QIITA_CONFIG_FP` environment variable, that’s the file that Qiita will use.
 * The `[main]` section sets a `TEST_ENVIRONMENT` variable, which determines whether your system will be running unit tests or if it a demo/production system. You will want to set the value to TRUE if you are running the unit tests.
 
 **A note on data accumulation**: Qiita keeps data in the `BASE_DATA_DIR` as the system gets used. When you drop a Qiita environment and create a fresh testing environment, the “old” data that was generated from the previous environment should be **manually** deleted (or, at least, removed from the data directories in the `BASE_DATA_DIR`).
 
-###Unit tests
+### Unit tests
 
 Unit tests in Qiita are located inside the tests/test folder of every sub-module, for example `qiita_db/test/test_metadata_template.py`. These can be executed on a per-file basis or using `nosetests` from the base directory.
 
 During test creation make sure the test class is decorated with `@qiita_test_checker()` if database modifications are done during tests. This will automatically drop and rebuild the qiita schema after the entire test class has been executed. This requires to all the tests in a single class be independent of each other, so stochastic failures do not occur due to different test order execution.
 
 Coverage testing is in effect, so run tests using `nosetests --with-coverage [test_file.py]` to check what lines of new code in your pull request are not tested.
 
-###Documentation
+### Documentation
 
 The documentation for Qiita is maintained as part of this repository, under the
 `qiita_pet/support_files/doc` folder, for more information, see the README.md
 file in `qiita_pet/support_files/doc/README.md`.
 
-###Scripts
+### Scripts
 
 Scripts in Qiita are located inside the scripts directory, their actions will rely on the settings described in the Qiita config file, for example if you are dropping a database, the database that will be dropped is the one described by the `DATABASE` setting. The following is a list of the most commonly used commands during development:
 
 * `qiita-env make` will create a new environment (as specified by the Qiita config file).
 * `qiita-env drop` will delete the environment (as specified by the Qiita config file).
-* `qiita-env start_cluster qiita-general`, starts an IPython cluster named ‘qiita-general’. Normally you’ll want to wait a few seconds for the engines to start and become responsive (30-40 seconds depending on your system).
-* `qiita-env stop_cluster qiita-general`, terminates a cluster named ‘qiita-general’.
 * `qiita pet webserver start`, will start the Qiita web-application running on port 21174, you can change this using the `--port` flag, for example `--port=7532`.
 
-##Making Database Changes
+## Making Database Changes
 After the initial production release of Qiita, changes to the database schema will require patches; the database can no longer be dropped and recreated using the most recent schema because all the data would be lost! Therefore, patches must be applied instead.
 
-###Approach
+### Approach
 
 1. We keep "unpatched" versions of the SQL and DBS files in the repository
 2. We keep fully patched versions of the DBS and HTML files in the repository
 3. We keep a patch file for each patch as required in the `qiita_db/support_files/patches` directory. Note that **the patches will be applied in order based on the natural sort order of their filename** (e.g., `2.sql` will be applied before `10.sql`, and `10.sql` will be applied before `a.sql`)
 
-###Developer Workflow
+### Developer Workflow
 
 1. Load the fully patched DBS file (e.g., `qiita-db.dbs`) in [DBSchema](http://www.dbschema.com/)
 2. Make desired changes
@@ -101,17 +107,17 @@ After the initial production release of Qiita, changes to the database schema wi
 
 One drawback is that developers will need to have [DBSchema](http://www.dbschema.com/) to develop for this project.
 
-###Data Patches
+### Data Patches
 
 If you need to submit a patch that changes only data but does not alter the schema, you should still create a patch file with the next name (e.g., `2.sql`) with your changes. Note that a patch should *not* be created if the modifications do not apply to Qiita databases in general; data patches are only necessary in some cases, e.g., if the terms in an ontology change.
 
-###Python Patches
+### Python Patches
 
 Occasionally, SQL alone cannot effect the desired changes, and a corresponding python script must be run after the SQL patch is applied. If this is the case, a python file should be created in the `patches/python_patches` directory, and it should have the same basename as the SQL file. For example, if there is a patch `4.sql` in the `patches` directory, and this patch requires a python script be run after the SQL is applied, then the python file should be placed at `patches/python_patches/4.py`. Note that not every SQL patch will have a corresponding python patch, but every python patch will have a corresponding SQL patch.
 
 If in the future we discover a use-case where a python patch must be applied for which there *is no corresponding SQL patch*, then a blank SQL patch file will still need to be created.
 
-##SQL coding guidelines
+## SQL coding guidelines
 Since the `qiita_db` code contains a mixture of python code and SQL code, here are some coding guidelines to add SQL code to Qiita:
 
 1. Any SQL keyword should be written uppercased:

INSTALL.md

@@ -60,6 +60,7 @@ Install the non-python dependencies
 
 * [PostgreSQL](http://www.postgresql.org/download/) (minimum required version 9.3.5, we have tested most extensively with 9.3.6)
 * [redis-server](http://redis.io) (we have tested most extensively with 2.8.17)
+* [webdis] (https://github.com/nicolasff/webdis) (latest version should be fine but we have tested the most with 9ee6fe2 - Feb 6, 2016)
 
 There are several options to install these dependencies depending on your needs:
 
@@ -87,6 +88,27 @@ brew update
 brew install homebrew/versions/redis28
 ```
 
+### webdis
+
+Note that this is the only package that assumes that Qiita is already installed (due to library dependencies). Also, that the general suggestion is to have 2 redis servers running, one for webdis/redbiom and the other for Qiita. The reason for multiple redis servers is so that the redbiom cache can be flushed without impacting the operation of the qiita server itself.
+
+The following instructions install, compile and pre-populates the redbiom redis DB so we assume that redis is running on the default port and that Qiita is fully installed as the redbiom package is installed with Qiita.
+
+```
+git clone https://github.com/nicolasff/webdis
+pushd webdis
+make
+./webdis &
+popd
+# note that this assumes that Qiita is already installed
+fp=`python -c 'import qiita_db; print qiita_db.__file__'`
+qdbd=`dirname $fp`
+redbiom admin scripts-writable
+redbiom admin create-context --name "qiita-test" --description "qiita-test context"
+redbiom admin load-sample-metadata --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt
+redbiom admin load-sample-metadata-search --metadata ${qdbd}/support_files/test_data/templates/1_19700101-000000.txt
+redbiom admin load-sample-data --table ${qdbd}/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test --tag 1
+```
 
 Install Qiita development version and its python dependencies
 -------------------------------------------------------------
@@ -110,11 +132,6 @@ Install Qiita (this occurs through setuptools' `setup.py` file in the qiita dire
 pip install -e . --process-dependency-links
 ```
 
-Install the development version of moi:
-```bash
-pip install https://github.com/biocore/mustached-octo-ironman/archive/master.zip --no-deps
-```
-
 At this point, Qiita will be installed and the system will start. However,
 you will need to install plugins in order to process any kind of data. For a list
 of available plugins, visit the [Qiita Spots](https://github.com/qiita-spots)
@@ -145,22 +162,22 @@ Set your `QIITA_CONFIG_FP` environment variable to point to that file (into `.ba
 
 ```bash
   echo "export QIITA_CONFIG_FP=$HOME/.qiita_config_test.cfg" >> ~/.bashrc
-  echo "export MOI_CONFIG_FP=$HOME/.qiita_config_test.cfg" >> ~/.bashrc
   source ~/.bashrc
   # Re-enable conda environment for qiita
   source activate qiita
 ```
 
-Setup ipython profile for qiita:
+Next, make a test environment:
 
 ```bash
-ipython profile create qiita-general --parallel
+qiita-env make --no-load-ontologies
 ```
 
-Next, make a test environment:
+Finally, redbiom relies on the REDBIOM_HOST environment variable to set the URL to query. By default is set to Qiita redbiom public repository. To change it you could do:
+
 
 ```bash
-qiita-env make --no-load-ontologies
+export REDBIOM_HOST=http://my_host.com:7379
 ```
 
 ## Start Qiita
@@ -173,12 +190,6 @@ Next, start redis server (the command may differ depending on your operating sys
 redis-server
 ```
 
-Start the IPython cluster:
-
-```bash
-qiita-env start_cluster qiita-general && sleep 30
-```
-
 Start the qiita server:
 
 ```bash

qiita_core/configuration_manager.py

@@ -58,18 +58,6 @@ class ConfigurationManager(object):
         The host where the database lives
     port : int
         The port used to connect to the postgres database in the previous host
-    ipyc_demo : str
-        The IPython demo cluster profile
-    ipyc_demo_n : int
-        The size of the demo cluster
-    ipyc_reserved : str
-        The IPython reserved cluster profile
-    ipyc_reserved_n : int
-        The size of the reserved cluster
-    ipyc_general : str
-        The IPython general cluster profile
-    ipyc_general_n : int
-        The size of the general cluster
     smtp_host : str
         The SMTP host from which mail will be sent
     smtp_port : int
@@ -145,7 +133,7 @@ def __init__(self):
             config.readfp(conf_file)
 
         _required_sections = {'main', 'redis', 'postgres', 'smtp', 'ebi',
-                              'ipython', 'portal'}
+                              'portal'}
         if not _required_sections.issubset(set(config.sections())):
             missing = _required_sections - set(config.sections())
             raise MissingConfigSection(', '.join(missing))
@@ -155,7 +143,6 @@ def __init__(self):
         self._get_postgres(config)
         self._get_redis(config)
         self._get_ebi(config)
-        self._get_ipython(config)
         self._get_vamps(config)
         self._get_portal(config)
 
@@ -283,10 +270,6 @@ def _get_ebi(self, config):
         self.ebi_center_name = sec_get('EBI_CENTER_NAME')
         self.ebi_organization_prefix = sec_get('EBI_ORGANIZATION_PREFIX')
 
-    def _get_ipython(self, config):
-        self.ipython_contexts = config.get('ipython', 'context').split(',')
-        self.ipython_default = config.get('ipython', 'default')
-
     def _get_vamps(self, config):
         self.vamps_user = config.get('vamps', 'USER')
         self.vamps_pass = config.get('vamps', 'PASSWORD')