fix-3064

.github/workflows/qiita-ci.yml

@@ -0,0 +1,208 @@
+# name: Qiita CI
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+
+jobs:
+  # derived from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml
+  main:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        include:
+          - cover_package: "qiita_db"
+          - cover_package: "qiita_pet qiita_core qiita_ware"
+
+    services:
+      postgres:
+        # Docker Hub image
+        image: postgres:9.5
+        env:
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          COVER_PACKAGE: ${{ matrix.cover_package }}
+
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # based on https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L44-L72
+          - 5432/tcp
+
+    steps:
+      # Downloads a copy of the code in your repository before running CI tests
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Setup for conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: 3.6
+
+      - name: Basic dependencies install
+        env:
+          COVER_PACKAGE: ${{ matrix.cover_package }}
+        shell: bash -l {0}
+        run: |
+          echo "Testing: " $COVER_PACKAGE
+
+          # pull out the port so we can modify the configuration file easily
+          pgport=${{ job.services.postgres.ports[5432] }}
+          sed -i "s/PORT = 5432/PORT = $pgport/" qiita_core/support_files/config_test.cfg
+
+          # PGPASSWORD is read by pg_restore, which is called by the build_db process.
+          export PGPASSWORD=postgres
+
+          # Setting up main qiita conda environment
+          conda config --add channels conda-forge
+          conda create -q --yes -n qiita python=3.6 pip libgfortran numpy nginx cython redis
+          conda activate qiita
+          pip install sphinx sphinx-bootstrap-theme nose-timer codecov Click
+
+          # Configuring SSH
+          cp /etc/ssh/sshd_config sshd_config
+          echo "RSAAuthentication yes" > sshd_config
+          echo "PubkeyAuthentication yes" > sshd_config
+          echo "StrictModes no" > sshd_config
+          sudo mv sshd_config /etc/ssh/sshd_config
+          sudo systemctl restart ssh
+
+      - name: Webdis install
+        shell: bash -l {0}
+        run: |
+          sudo apt-get -y install libevent-dev
+          git clone https://github.com/nicolasff/webdis
+          cd webdis
+          make
+
+      - name: Main install
+        shell: bash -l {0}
+        run: |
+          conda activate qiita
+          export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+          export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          export REDBIOM_HOST="http://localhost:7379"
+          pip install . --no-binary redbiom
+          pwd
+          mkdir ~/.qiita_plugins
+
+      - name: Install plugins
+        shell: bash -l {0}
+        run: |
+          wget https://data.qiime2.org/distro/core/qiime2-2019.4-py36-linux-conda.yml
+          conda env create -q -n qtp-biom --file qiime2-2019.4-py36-linux-conda.yml
+          rm qiime2-2019.4-py36-linux-conda.yml
+          export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+          export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          export REDBIOM_HOST="http://localhost:7379"
+          conda activate qtp-biom
+          pip install -U pip
+          pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
+          pip install https://github.com/qiita-spots/qtp-biom/archive/master.zip
+          configure_biom --env-script "source /home/runner/.profile; conda activate qtp-biom" --server-cert $QIITA_SERVER_CERT
+
+      - name: Starting services
+        shell: bash -l {0}
+        run: |
+          conda activate qiita
+          export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+          export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          export REDBIOM_HOST="http://localhost:7379"
+
+          echo "1. Setting up redis"
+          redis-server --daemonize yes --port 7777
+          redis-server --daemonize yes --port 6379
+
+          echo "2. Starting webdis"
+          pushd webdis
+          ./webdis &
+          popd
+
+          echo "3. Starting redbiom: " $REDBIOM_HOST
+          curl -s http://localhost:7379/FLUSHALL > /dev/null
+          redbiom --version
+          redbiom admin scripts-writable
+          redbiom admin create-context --name "qiita-test" --description "qiita-test context"
+          redbiom admin load-sample-metadata --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+          redbiom admin load-sample-metadata-search --metadata `pwd`/qiita_db/support_files/test_data/templates/1_19700101-000000.txt
+          redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table.biom --context qiita-test --tag 4
+          redbiom admin load-sample-data --table `pwd`/qiita_db/support_files/test_data/processed_data/1_study_1001_closed_reference_otu_table-for_redbiom_tests.biom --context qiita-test --tag 5
+
+          echo "4. Setting up nginx"
+          mkdir -p /usr/share/miniconda/envs/qiita/var/run/nginx/
+          nginx -c ${PWD}/qiita_pet/nginx_example.conf
+
+          echo "5. Setting up qiita"
+          conda activate qiita
+          qiita-env make --no-load-ontologies
+          qiita-test-install
+          qiita plugins update
+
+          echo "6. Starting supervisord => multiple qiita instances"
+          supervisord -c ${PWD}/qiita_pet/supervisor_example.conf
+          sleep 10
+          cat /tmp/supervisord.log
+
+          echo "7. Starting plugins"
+          conda deactivate
+          conda activate qtp-biom
+          export QIITA_CLIENT_DEBUG_LEVEL=DEBUG
+          start_biom https://localhost:8383 register ignored
+          conda deactivate
+
+          echo "8. Setting up SSH"
+          ssh-keygen -t rsa-sha2-256 -b 4096 -N '' -f $PWD/qiita_ware/test/test_data/test_key
+          mkdir ~/.ssh/
+          cp $PWD/qiita_ware/test/test_data/test_key* ~/.ssh/
+          cat ~/.ssh/test_key.pub > ~/.ssh/authorized_keys
+          chmod 600 $PWD/qiita_ware/test/test_data/test_key*
+          chmod 600 ~/.ssh/*
+          chmod 700 ~/.ssh/
+          scp -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:$PWD/qiita_ware/test/test_data/random_key $PWD/qiita_ware/test/test_data/random_key_copy
+
+      - name: Main tests
+        shell: bash -l {0}
+        env:
+          COVER_PACKAGE: ${{ matrix.cover_package }}
+        run: |
+          conda activate qiita
+          bash -c 'source /home/runner/.profile; conda activate qtp-biom; start_biom --help'
+          export QIITA_SERVER_CERT=`pwd`/qiita_core/support_files/server.crt
+          export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
+          export REDBIOM_HOST="http://localhost:7379"
+          nosetests $COVER_PACKAGE --with-doctest --with-coverage --with-timer -v --cover-package=$COVER_PACKAGE -e 'test_submit_EBI_parse_EBI_reply_failure' -e 'test_full_submission'
+
+          # killing the qiita server to run the next commands
+          QIITA_PID=`cat /tmp/supervisord.pid`
+          kill $QIITA_PID
+          sleep 10
+          if [[ "$COVER_PACKAGE" == *"qiita_db"* ]]; then test_data_studies/commands.sh; all-qiita-cron-job; fi
+
+      - uses: codecov/codecov-action@v1
+        with:
+         file: codecov.yml
+         fail_ci_if_error: true
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - name: flake8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+    - name: install dependencies
+      run: python -m pip install --upgrade pip
+    - name: Check out repository code
+      uses: actions/checkout@v2
+    - name: lint
+      run: |
+        pip install -q flake8
+        flake8 qiita_* setup.py scripts/qiita*

README.rst

@@ -71,7 +71,7 @@ future.
   existing studies by submitting a fix proposals to the authors of the study.
 
 
-.. |Build Status| image:: https://travis-ci.org/qiita-spots/qiita.png?branch=master
-   :target: https://travis-ci.org/qiita-spots/qiita
+.. |Build Status| image:: https://github.com/qiita-spots/qiita/actions/workflows/qiita-ci.yml/badge.svg
+   :target: https://github.com/qiita-spots/qiita/actions/workflows/qiita-ci.yml
 .. |Coverage Status| image:: https://codecov.io/gh/qiita-spots/qiita/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/qiita-spots/qiita

qiita_core/support_files/config_test.cfg

@@ -26,16 +26,16 @@ REQUIRE_APPROVAL = True
 BASE_URL = https://localhost:8383
 
 # Download path files
-UPLOAD_DATA_DIR = /home/travis/miniconda3/envs/qiita/lib/python3.6/site-packages/qiita_db/support_files/test_data/uploads/
+UPLOAD_DATA_DIR = /home/runner/work/qiita/qiita/qiita_db/support_files/test_data/uploads/
 
 # Working directory path
-WORKING_DIR = /home/travis/miniconda3/envs/qiita/lib/python3.6/site-packages/qiita_db/support_files/test_data/working_dir/
+WORKING_DIR = /home/runner/work/qiita/qiita/qiita_db/support_files/test_data/working_dir/
 
 # Maximum upload size (in Gb)
 MAX_UPLOAD_SIZE = 100
 
 # Path to the base directory where the data files are going to be stored
-BASE_DATA_DIR = /home/travis/miniconda3/envs/qiita/lib/python3.6/site-packages/qiita_db/support_files/test_data/
+BASE_DATA_DIR = /home/runner/work/qiita/qiita/qiita_db/support_files/test_data/
 
 # Valid upload extension, comma separated. Empty for no uploads
 VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual
@@ -118,10 +118,10 @@ HOST = localhost
 PORT = 5432
 
 # The password to use to connect to the database
-PASSWORD =
+PASSWORD = postgres
 
 # The postgres password for the admin_user
-ADMIN_PASSWORD =
+ADMIN_PASSWORD = postgres
 
 # ----------------------------- Torque settings -----------------------------
 [torque]

qiita_pet/nginx_example.conf

@@ -8,6 +8,7 @@ http {
         server localhost:21174;
         server localhost:21175;
         server localhost:21176;
+        server localhost:21177;
     }
 
     # listening to 8080 and redirecting to https
@@ -23,8 +24,8 @@ http {
         merge_slashes off;
         ssl          on;
 
-        ssl_certificate	/home/travis/miniconda3/envs/qiita/lib/python3.6/site-packages/qiita_core/support_files/server.crt;
-        ssl_certificate_key /home/travis/miniconda3/envs/qiita/lib/python3.6/site-packages/qiita_core/support_files/server.key;
+        ssl_certificate	/home/runner/work/qiita/qiita/qiita_core/support_files/server.crt;
+        ssl_certificate_key /home/runner/work/qiita/qiita/qiita_core/support_files/server.key;
 
         ssl_session_timeout  5m;
 

qiita_pet/supervisor_example.conf

@@ -10,7 +10,7 @@ pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid
 files=/home/travis/miniconda3/envs/qiita/bin
 
 [group:qiita_all]
-programs=qiita_workers_0,qiita_workers_1,qiita_master  ; each refers to 'x' in [program:x] definitions
+programs=qiita_workers_0,qiita_workers_1,qiita_workers_2,qiita_master  ; each refers to 'x' in [program:x] definitions
 
 [program:qiita_master]
 command=qiita pet webserver --no-build-docs start --port 21174 --master
@@ -26,3 +26,8 @@ numprocs=1                    ; number of processes copies to start (def 1)
 command=qiita pet webserver --no-build-docs start --port 21176
 process_name=%(program_name)s ; process_name expr (default %(program_name)s)
 numprocs=1                    ; number of processes copies to start (def 1)
+
+[program:qiita_workers_2]
+command=qiita pet webserver --no-build-docs start --port 21177
+process_name=%(program_name)s ; process_name expr (default %(program_name)s)
+numprocs=1                    ; number of processes copies to start (def 1)