target galaxy release_23.0 (#32)

* target galaxy release_23.0

* Update test_playbook.yaml

* Update requirements.yml

* Update playbook.yml

* Update 000_cross_env_vars

* create a dbserver.yml "host"

* reorganize playbook in 2 hosts dbserver and all

* Update 000_cross_env_vars

* Update hosts

* add - 'virtualenv' in dependencies. Don't get why it's not in the GNT ansible training !

* change miniconda version !

and undo miniconda_manage_dependencies = false (commented)

* switch to job_conf.yml structure

* switch uptream galaxy on gunicorn.sock

* Update ansible.cfg

* tune group_vars galaxy and nginx

* Update .gitignore

* switch to ansible 3.0 and bioblend 1.2.0

* upgrade Docker role to release_23.0 and lint dev_gce

* add psql user galaxy in Dockerfile

* create galaxy psql db before ansible playbook

* remove nginx post task and tweek ansible-playbook

remove -c local from the ansible-playbook command in Dockerfile

* galaxy_manage_systemd is false by default

* Fixing bugged handler that crashes in docker

* upgrade dev_mississippi env to release_23.0

* let job_conf.yml as is in files

* Update job_conf.yml

* Update galaxy.j2

* remove job_conf and update slurm group_vars

* conf_job.yml Mississippi

* slurm conf of Mississippi

* Update mississippi_tool_list.yml

* Update Mississippi hosts file

* fix miniconda_channels Mississippi

* nginx Mississippi configuration

* Update dev_mississippi slurm group_vars

* adjust Mississippi slurm group_vars

* Update Mississippi job_conf.yml

* create an uptime service to replace uptime supervisor

* Update .gitignore

* Update Conect hosts

* Conect job_conf.yml

* Forgot create a simlink to dbservers.yml for Mississippi

So strangely enough it may be not required !

* conect template nginx/galaxy.j2

* Fix dev_mississippi welcome page

* fix mississippi_dev.png target location

* Update welcome page of Mississippi-2

* Update conect_tool_list.yml

* Update artbio_tool_list.yml

* Update ARTbio hosts

It seems that there was a bug in this config with [Conect] instead of [ARTbio] !

* Update ARTbio galaxy.j2

* Update artbio.snv.jussieu.fr.crt

There was a need to re-import the updated crt into this already old branch !

* Update welcome.html

test fastqc under java destination

* tune fastqc in Mississippi job_conf.yml

* 8 workers for Mississippi !

* 8 workers for ARTbio !

* test conda 3.11 for ARTbio

* Change the default destination of ARTbio to slurm cluster_1

* Change the default destination of Conect to slurm cluster_1

* Change the default destination of Mississippi[2] to slurm cluster_1

.github/workflows/test_playbook.yaml

@@ -39,10 +39,10 @@ jobs:
         python3 -m pip install -U pip setuptools
 
 
-    - name: Install ansible 2.10.1 and bioblend in ubuntu 20.04
+    - name: Install ansible 3.0 and bioblend in ubuntu 20.04
       run: |
-        python3 -m pip install ansible==2.10.1
-        python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.0.0.zip pytest
+        python3 -m pip install ansible==3.0
+        python3 -m pip install --ignore-installed https://github.com/galaxyproject/bioblend/archive/refs/tags/v1.2.0.zip pytest
     
     - name: Display pip python and ansible settings
       run: |
@@ -73,10 +73,12 @@ jobs:
                          --skip-tags galaxy_build_client \
                          playbook.yml
 
-    - name: Sleep for 15 seconds and check galaxyctl status
+    - name: Sleep for 30 secs and check galaxyctl status
       run: |
-        echo "waiting 15 sec" && sleep 15
-        sudo su - galaxy -c "source galaxy/.venv/bin/activate && galaxyctl status"
+        sudo galaxyctl graceful
+        echo "waiting 30 sec" && sleep 30
+        sudo galaxyctl status
+# sudo su - galaxy -c "source galaxy/.venv/bin/activate && galaxyctl status"
 
     - name: check Galaxy API is alive
       run: |

.gitignore

@@ -2,142 +2,10 @@
 .DS_Store
 *.pyc
 docs/html
-
-roles/ansible-slurm/
-roles/galaxyproject.galaxy/
-roles/galaxyproject.nginx/
-roles/galaxyproject.postgresql/
-roles/galaxyproject.postgresql_objects/
-roles/geerlingguy.pip/
-roles/uchida.miniconda/
-roles/install.galaxy-tools/
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
+.vault-password.txt
+roles/galaxyproject.galaxy
+roles/galaxyproject.miniconda
+roles/galaxyproject.nginx
+roles/galaxyproject.postgresql
+roles/galaxyproject.postgresql_objects
+roles/galaxyproject.slurm

Dockerfile.galaxyxpand

@@ -17,7 +17,7 @@ RUN apt update && \
     postgresql-12 postgresql-client-12 postgresql-contrib-12 \
     locales dirmngr && apt clean
 
-RUN python3 -m pip install -U pip && python3 -m pip install ansible==2.10.7 && \
+RUN python3 -m pip install -U pip && python3 -m pip install ansible==3.0 && \
     python3 -m pip install -U cryptography pyyaml
 
 # Run the rest of the commands as the ``postgres`` user created by the ``postgres-12``
@@ -37,7 +37,9 @@ USER postgres
 
 RUN    /etc/init.d/postgresql start &&\
     psql --command "CREATE USER root WITH SUPERUSER PASSWORD 'galaxy';" && \
-    createdb -O root rootdb
+    createdb -O root rootdb && \
+    psql --command "CREATE USER galaxy WITH PASSWORD 'galaxy';" && \
+    createdb -O galaxy galaxy
 
 # Adjust PostgreSQL configuration so that remote connections to the
 # database are possible.
@@ -80,11 +82,16 @@ WORKDIR /setup
 ENV LC_ALL=en_US.UTF-8 \
     LANG=en_US.UTF-8
 
+# in next step, a faulty handler that triggered systemctl is deleted
+# before running the playbook
+
 RUN echo "remote_tmp = /setup/.ansible/tmp" >> ansible.cfg && \
     service postgresql start && \
     sed -i '/slurm-wlm/d' playbook.yml && sed -i '/slurm-drmaa-dev/d' playbook.yml && \
     ansible-galaxy install -r requirements.yml -p roles -f && \
-    ansible-playbook -i environments/Docker/hosts -c local playbook.yml
+    head -n -4 roles/galaxyproject.galaxy/handlers/main.yml > tmp.txt &&\
+    mv tmp.txt roles/galaxyproject.galaxy/handlers/main.yml &&\
+    ansible-playbook -i environments/Docker/hosts playbook.yml
 
 ONBUILD  WORKDIR  /setup
 ONBUILD  COPY  .  /setup

ansible.cfg

@@ -1,8 +1,12 @@
 [defaults]
 interpreter_python = /usr/bin/python3
 inventory = ./environments/dev_gce/hosts
-retry_files_enabled = False
-display_skipped_hosts = False
-stdout_callback = community.general.yaml
-bin_ansible_callbacks = True # Use the stdout_callback when running ad-hoc commands.
+# Use the YAML callback plugin.
+stdout_callback = yaml
+# Use the stdout_callback when running ad-hoc commands.
+bin_ansible_callbacks = True
 # vault_password_file = ../ansible_vault_password
+
+# Show diffs of changes
+[diff]
+always = True

environments/000_cross_env_vars

@@ -1,29 +1,17 @@
 ---
 # Python 3 support
-pip_virtualenv_command: /usr/bin/python3 -m virtualenv
-pip_package: python3-pip
+pip_virtualenv_command: /usr/bin/python3 -m venv
 
-# PostgreSQL
-postgresql_objects_users:
-  - name: galaxy
-postgresql_objects_databases:
-  - name: galaxy
-    owner: galaxy
-# PostgreSQL Backups
-postgresql_backup_dir: /data/backups
-postgresql_backup_local_dir: "{{ '~postgres' | expanduser }}/backups"
+# Common variables needed by all hosts
+galaxy_user_name: galaxy
+galaxy_db_name: galaxy
 
-# galaxy
-galaxy_config_perms: 0664
 
-# when transitionning from galaxykickstart to galaxyXpand set to 'yes'
-galaxykickstart_to_gravity: no
+# galaxy (check requirement)
+galaxy_config_perms: 0664
 
 # to install uptime set to 'yes'
 install_uptime: no
 
-# to not install postgresql set to 'no"
-install_postgresql: yes
-
 # to not install slurm set to 'no'
 install_slurm: yes