Ansible deploy dev rebase dev 2025

.gitignore

@@ -2,3 +2,4 @@
 docker/.env
 .output/
 */priv/static
+todo

ansible/.gitignore

@@ -0,0 +1,4 @@
+.password
+.vault_password
+/data
+!/data/.gitkeep

ansible/README.md

@@ -0,0 +1,67 @@
+# Gup-admin playbooks
+
+## Installation
+To install ub-ansible-deploy role and other dependencies required by this playbook run `ansible-galaxy install -r requirements.yml`
+To update dependencies run the same command with the -f flag to force reinstalltion.
+
+## Configuration
+
+### Add new ansible host
+
+For a new host, create `inventory/<host-alias>.yml` containing:
+
+```yaml
+---
+all:
+  hosts:
+    <host-alias>:
+      ansible_host: <host-url>
+      ansible_user: apps
+```
+
+Set host specic variables in `host_vars/<host-alias>/vars.yml`.
+
+Set encrypted host specific variables in `host_vars/<host-alias>/vault.yml`, prefixed with `vault_`.
+
+To create the encrypted vault.yml file run:
+`ansible-vault create --vault-password-file .vault_password host_vars/<host-alias>/vault.yml`
+Replace `create` with `edit` to edit the file.
+
+Vault variables should then be used in the plain text file like:
+`variable_name: "{{ vault_variable_name }}"`
+
+This way they are searchable with tools such as grep etc.
+
+Set variables shared between hosts in `group_vars/all/vars.yml`.
+Set encrypted shared variables, such as common api keys etc, in `group_vars/all/vault.yml`, prefixed with `vault_` and refered to in `group_vars/all/vars.yml` as shown above.
+
+### Docker .env and secrets.env configuration
+To set variables in `.env` and `secrets.env` there are a number of special files:
+
+#### vars/default_env.yml
+Environment variables shared between all hosts. Existing values in .env will be replaced with variables defined here.
+
+#### vars/default_secret_env.yml
+Secret environment variables shared between all hosts. secret.env.example will be copied to secret.env and existing values will be replaced with values defined here.
+
+#### vars/\<host-alias\>/env.yml
+Host specific environment variables, these will be merged in with the default variables.
+
+#### vars/\<host-alias\>/secret_env.yml
+Host specific secret environment variables, these will be merged in with the default secret variables.
+
+### Apache configuration
+`<file-name>.conf.j2` files in `templates/sites/` will be deployed to /etc/gub-apache2/sites as `<file-name>-<host-alias.conf`, except for production where the host-alias suffix is not included.
+
+### Cron configuration
+`<file-name>,j2` files in `templates/cron.d/ will be deployed to `/etc/cron.d/<file-name>`.
+
+## Deployment
+- Save vault password in `.vault_password`
+- Run ./run-playbook.sh <host> deploy replacing \<host\> with host alias name, for example `lab`.
+- To use ansible-playbook command directly run `ansible-playbook -i inventory/<host>.yml --vault-password-file .vault_password deploy.yml` 
+
+(The -C flag can be used to run the playbook without performing and changes on the target server.)
+
+## ./run-playbook.sh
+Helper script for running playbook. To use, run `./run-playbook.sh <playbook> <host> <optiona-extra-arguments>`.

ansible/deploy.yml

@@ -0,0 +1,67 @@
+---
+- name: Deploy
+  hosts: all
+  tasks:
+
+  - name: Include default environment variables
+    ansible.builtin.include_vars:
+      file: default_env.yml
+      name: env_variables
+
+  - name: Include environment variables
+    ansible.builtin.include_vars:
+      file: "{{ inventory_hostname }}/env.yml"
+      name: env_variables
+      hash_behaviour: merge
+
+  - name: Include default secret environment variables
+    ansible.builtin.include_vars:
+      file: default_secret_env.yml
+      name: secret_env_variables
+
+  - name: Include secret environment variables
+    ansible.builtin.include_vars:
+      file: "{{ inventory_hostname }}/secret_env.yml"
+      name: secret_env_variables
+      hash_behaviour: merge
+
+  - name: Ensure data directory
+    become: yes
+    ansible.builtin.file:
+      path: "{{ data_dir }}"
+      state: directory
+      owner: "{{ ansible_user }}"
+      group: "{{ ansible_user }}"
+      mode: '755'
+
+  - name: Ensure data sub-directies
+    become: yes
+    ansible.builtin.file:
+      path: "{{ item }}"
+      state: directory
+      owner: "{{ ansible_user }}"
+      group: "{{ ansible_user }}"
+      mode: '755'
+    loop:
+      - "{{ db_data_dir }}"
+      - "{{ index_manager_db_data_dir }}"
+      - "{{ backend_source_data_dir }}"
+      - "{{ backend_scopus_normalised_data_dir }}"
+      - "{{ scopus_normalised_json_files_data_dir }}"
+      - "{{ scopus_scripts_data_dir }}"
+
+  - name: Ensure elasticsearch directories
+    become: yes
+    ansible.builtin.file:
+      path: "{{ item }}"
+      state: directory
+      owner: '1000'
+      group: '1000'
+      mode: '755'
+    loop:
+      - "{{ elasticsearch_data_dir }}"
+      - "{{ kibana_data_dir }}"
+
+  - name: Deploy app
+    include_role:
+      name: ub-ansible-deploy

ansible/export-backend-db.yml

@@ -0,0 +1,9 @@
+---
+- name: Export database
+  hosts: all
+  roles:
+  - role: ub-ansible-export-db
+    vars:
+      db_service: admin-db
+      database_variant: postgres
+      db_dump_filename: backend-database.sql

ansible/export-db.yml

@@ -0,0 +1,3 @@
+---
+- import_playbook: export-backend-db.yml
+- import_playbook: export-index-manager-db.yml

ansible/export-index-manager-db.yml

@@ -0,0 +1,9 @@
+---
+- name: Export index manager database
+  hosts: all
+  roles:
+  - role: ub-ansible-export-db
+    vars:
+      db_service: index-manager-db
+      database_variant: postgres
+      db_dump_filename: index-manager-database.sql

ansible/group_vars/all.yml

@@ -0,0 +1,14 @@
+---
+docker_git_repo_url: https://github.com/ub-digit/gup-admin.git
+docker_directory: './docker' # Default is './docker'
+app_name: gup-admin
+
+data_dir: /data/gup-admin
+db_data_dir: /data/gup-admin/db
+index_manager_db_data_dir: /data/gup-admin/index-manager-db
+backend_source_data_dir: /data/gup-admin/_source
+backend_scopus_normalised_data_dir: /data/gup-admin/scopus-normalised
+scopus_normalised_json_files_data_dir: /data/gup-admin/scopus-files
+scopus_scripts_data_dir: /data/gup-admin/scopus-scripts
+elasticsearch_data_dir: /data/gup-admin/elasticsearch
+kibana_data_dir: /data/gup-admin/kibana

ansible/host_vars/lab/vars.yml

@@ -0,0 +1,11 @@
+---
+docker_git_revision: ansible-deploy
+git_revision: author-edit-017
+git_revision_gup_imports: release-2023.08-001
+frontend_host_port: 30410
+backend_host_port: 30415
+index_manager_backend_host_port: 30411
+elasticsearch_host_port: 30413
+kibana_host_port: 30416
+backend_hostname: "{{ app_name }}-server-lab.ub.gu.se"
+frontend_hostname: "{{ app_name }}-lab.ub.gu.se"

ansible/host_vars/lab/vault.yml

@@ -0,0 +1,21 @@
+$ANSIBLE_VAULT;1.1;AES256
+66343936666365316131383237653236613762306132343330666239616464313664666537373362
+3831393935383935666166633632353661383362333835640a393333636635373430613733636462
+35386666333466383362643232656361633739363831333362396539636236333437613733366630
+3032636564393535300a303261373030383337333034373137363461653937653161626533393530
+34323730636632653234386539383061303239633835333337373439316631666136396362333435
+31323330656233623933653264316566656264333531333038626462323032356532663035333132
+33613433643837636536353234393538386135386531646661386133636437623035336661363434
+31366239326261626139303638383164316661616531396532323939663534633466663264376534
+32386531366338396430353938373337323162363863323836336537663062393365316165363233
+32633431386633386365653936373932613238356336396239303833353335396330656562376464
+30386239653930636630336162626330356366326330653765393861393064326365306536356164
+33366636373137613666383361316532643633363938386264633666613163386466306134646461
+39333731306165616463373639643666613763363038633662343531353164326465633433323763
+34653666366336643533303835316136316338343861336435633536636330343034343262316464
+63356232323462613534643864396666356532646338383234303363306138666133386334623463
+61336639363835363766663432623033616136623733653566366335623263366139646661336464
+61393664343465353939643566663931616231383832383965373334653036396231613065326534
+32613862366266353330633564353432386532376533663062366433356636663462333066363839
+62653033303937323839313163313336336533643538356362363861623333643661396563656239
+61633630633734393866

ansible/host_vars/staging/vars.yml

@@ -0,0 +1,11 @@
+---
+docker_git_revision: ansible-deploy
+git_revision: author-edit-017
+git_revision_gup_imports: release-2023.08-001
+frontend_host_port: 20410
+backend_host_port: 20415
+index_manager_backend_host_port: 20411
+elasticsearch_host_port: 20413
+kibana_host_port: 20416
+backend_hostname: "{{ app_name }}-server-staging.ub.gu.se"
+frontend_hostname: "{{ app_name }}-staging.ub.gu.se"

ansible/host_vars/staging/vault.yml

@@ -0,0 +1,16 @@
+$ANSIBLE_VAULT;1.1;AES256
+31373861656434333539663237643533663838386432373938373662373630366236363061303735
+6439636337626663306665303561346461333431376639390a613331346435363961643832323234
+35666139623030613533323939366630663939653534666634623732613465386335396533623938
+3230323433303262340a363638626137396163636634663232383563386537323831323362376539
+32623634653762633934303161353462373663373332623538333635666535306630333334323831
+65333639356365636637326166393539653539306464373331633833663933353861633530663062
+31323663336532663233616335666462333034303366343434366537363263353434653338386632
+30633063323434623034313833613730663461613663386337313165643961393638613733393633
+38343238373864323937363031653532346662373334643339356664333432353936623633363136
+32323564613438656330346135333333333439376530666237623038653565646666303263303262
+33656334653163363233323731653065356366633566333234333833343832356434653064643062
+64643034646166313030643038306439303962356632363563383165653637383232326463616363
+62633739356239643331636636643532353433623132626664636166613166373033646537316237
+63666363303937306462326331363237616262666661313331373834303731643531623635633263
+313733313865636663386431323661363737

ansible/import-backend-db.yml

@@ -0,0 +1,9 @@
+---
+- name: Import database
+  hosts: all
+  roles:
+  - role: ub-ansible-import-db
+    vars:
+      db_service: admin-db
+      database_variant: postgres
+      db_dump_filename: backend-database.sql

ansible/import-db.yml

@@ -0,0 +1,3 @@
+---
+- import_playbook: import-backend-db.yml
+- import_playbook: import-index-manager-db.yml

ansible/import-devel-db.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+if [[ -z "$1" ]]; then
+    echo "Usage: $0 <target>"
+    echo "<target> must be on of staging, lab or production"
+    exit 0;
+fi
+
+targets="staging lab production"
+target=$1
+
+if [[ ! " $targets " =~ " $target " ]]; then
+    echo "<target> must be on of staging, lab, prod or production"
+    exit 1;
+fi
+
+#./run-playbook.sh $target export-db
+cd ../docker
+
+RUNNING=$(docker compose ps admin-db -q)
+if [[ -z "$RUNNING" ]]; then
+  echo "The service 'admin-db' is down, run docker compose up -d admin-db in docker directory"
+  exit 1;
+fi
+
+RUNNING=$(docker compose ps index-manager-db -q)
+if [[ -z "$RUNNING" ]]; then
+  echo "The service 'index-manager-db' is down, run docker compose up -d index-manager-db in docker directory"
+  exit 1;
+fi
+
+docker compose exec admin-db bash -c 'psql -d postgres -U $POSTGRES_USER -c "DROP DATABASE IF EXISTS $POSTGRES_DB;"'
+docker compose exec admin-db bash -c 'psql -d postgres -U $POSTGRES_USER -c "CREATE DATABASE $POSTGRES_DB;"'
+docker compose exec -T admin-db bash -c 'psql -d $POSTGRES_DB $POSTGRES_USER' < ../ansible/data/database.sql
+
+docker compose exec index-manager-db bash -c 'psql -d postgres -U $POSTGRES_USER -c "DROP DATABASE IF EXISTS $POSTGRES_DB;"'
+docker compose exec index-manager-db bash -c 'psql -d postgres -U $POSTGRES_USER -c "CREATE DATABASE $POSTGRES_DB;"'
+docker compose exec -T index-manager-db bash -c 'psql -d $POSTGRES_DB $POSTGRES_USER' < ../ansible/data/index-manager-database.sql
+
+echo "Databases has been imported"

ansible/import-index-manager-db.yml

@@ -0,0 +1,9 @@
+---
+- name: Import index manager database
+  hosts: all
+  roles:
+  - role: ub-ansible-import-db
+    vars:
+      db_service: index-manager-db
+      database_variant: postgres
+      db_dump_filename: index-manager-database.sql

ansible/init-devel.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+if [[ -z "$1" ]]; then
+    echo "Usage: $0 <target>"
+    echo "<target> must be on of staging, lab or production"
+    exit 0;
+fi
+
+targets="staging lab production"
+target=$1
+
+if [[ ! " $targets " =~ " $target " ]]; then
+    echo "<target> must be on of staging, lab, prod or production"
+    exit 1;
+fi
+
+./run-playbook.sh $target export-db
+
+cd ../docker
+./mkdirs
+cp ../ansible/data/backend-database.sql ./db-postgres-initdb.d/database.sql
+cp ../ansible/data/index-manager-database.sql ./index-manager-db-postgres-initdb.d/database.sql
+
+echo "Database dump has been copied to postgres-initdb.d directory"

ansible/inventory/lab.yml

@@ -0,0 +1,6 @@
+---
+all:
+  hosts:
+    lab:
+      ansible_host: gup-admin-lab.ub.gu.se
+      ansible_user: apps

ansible/inventory/staging.yml

@@ -0,0 +1,6 @@
+---
+all:
+  hosts:
+    lab:
+      ansible_host: gup-admin-staging.ub.gu.se
+      ansible_user: apps

ansible/requirements.yml

@@ -0,0 +1,3 @@
+---
+- src: https://github.com/ub-digit/ub-ansible-deploy.git
+  version: master

ansible/run-playbook.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [[ -z "$2" ]]; then
+    echo "Usage: $0 <target> <playbook> <extra playbook arguments>"
+    echo "<target> must be on of staging, lab, production and <playbook> is a playbook without the .yml extension"
+    exit 0;
+fi
+
+targets="staging lab production"
+target=$1
+playbook=$2
+
+if [[ ! -e "./$playbook.yml" ]]; then
+    echo "Playbook $playbook.yml does not exist"
+    exit 1;
+fi
+
+if [[ ! " $targets " =~ " $target " ]]; then
+    echo "<target> must be on of staging, lab, prod or production"
+    exit 1;
+fi
+
+# Shift so that $@ contains all remaining arguments
+shift;
+shift;
+
+set -x
+ansible-playbook --vault-password-file .vault_password -i inventory/$target.yml  $playbook.yml $@