Move secret configuration for postgresql cluster to a seperate playbook

ansible/inventory/offline/group_vars/postgresql/postgresql.yml

@@ -5,8 +5,18 @@ postgresql_conf_dir: /etc/postgresql/{{ postgresql_version }}/main
 
 # repmgr HA configuration
 repmgr_user: repmgr
-repmgr_password: "securepassword"
 repmgr_database: repmgr
+# Note: repmgr_password is NOT defined here - it's dynamically set by postgresql-secrets.yml
+#       The password is fetched from K8s secret or auto-generated during deployment.
+#       To manually pre-create: kubectl create secret generic repmgr-postgresql-secret -n <namespace> --from-literal=password=<your-password>
+
+# Kubernetes Secret configuration for repmgr
+repmgr_secret_name: "repmgr-postgresql-secret"
+repmgr_namespace: "{{ wire_namespace | default('default') }}"
+
+# Kubernetes Secret configuration for wire-server PostgreSQL user
+# This is managed alongside repmgr credentials in postgresql-secrets.yml
+wire_pg_secret_name: "wire-postgresql-external-secret"
 
 # Node configuration for repmgr
 repmgr_node_config:

ansible/postgresql-deploy.yml

@@ -1,41 +1,60 @@
+# ===================================================================
+# PostgreSQL Deployment Pipeline
+# ===================================================================
+# This playbook orchestrates PostgreSQL cluster deployment.
+# Playbooks run in strict order due to dependencies.
+#
+# Dependency Chain:
+#   secrets → primary → replica → verify → wire-setup → monitoring
+#   (cleanup/install are independent but should run first)
+#
+# Tag Strategy:
+#   - Use --skip-tags to exclude specific steps
+#   - Do NOT use --tags to run individual steps (will break dependencies)
+#   - Recommended: Run full playbook or use skip-tags
+#
+# Usage Examples:
+#   Full deployment(Cleans up the previous state if any, sets up HA pg cluster, deploy the guard against split-brain issues and sets up wire-server DB):    ansible-playbook -i <inventory_file> postgresql-deploy.yml
+#   Skip cleanup(cleans up the previous repmgr state and re-initializes the database as new state):       ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags cleanup
+#   Skip monitoring(In general this step should not be skipped as it sets up the guard against split-brain issues):    ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags monitoring
+#   Skip wire-setup (sets up the wire-server database and user):    ansible-playbook -i <inventory_file> postgresql-deploy.yml --skip-tags wire-setup
+#
+#
+# To run from a specific point, comment out earlier playbooks.
+# ===================================================================
+
 - name: Clean previous deployment state
   import_playbook: postgresql-playbooks/clean_existing_setup.yml
-  tags:
-    - postgresql
-    - cleanup
+  tags: [postgresql, cleanup]
 
 - name: Install PostgreSQL packages
   import_playbook: postgresql-playbooks/postgresql-install.yml
-  tags:
-    - postgresql
-    - install
+  tags: [postgresql, install]
+
+- name: Setup PostgreSQL passwords from Kubernetes Secrets
+  import_playbook: postgresql-playbooks/postgresql-secrets.yml
+  tags: [postgresql, secrets]
+  # Sets facts: repmgr_password, wire_pass (required by all following playbooks)
 
 - name: Deploy PostgreSQL primary node
   import_playbook: postgresql-playbooks/postgresql-deploy-primary.yml
-  tags:
-    - postgresql
-    - primary
+  tags: [postgresql, primary]
+  # Requires: repmgr_password
 
 - name: Deploy PostgreSQL replica nodes
   import_playbook: postgresql-playbooks/postgresql-deploy-replica.yml
-  tags:
-    - postgresql
-    - replica
+  tags: [postgresql, replica]
+  # Requires: repmgr_password
 
 - name: Verify PostgreSQL deployment
   import_playbook: postgresql-playbooks/postgresql-verify-HA.yml
-  tags:
-    - postgresql
-    - verify
+  tags: [postgresql, verify]
 
 - name: Setup wire-server postgresql database and user
   import_playbook: postgresql-playbooks/postgresql-wire-setup.yml
-  tags:
-    - postgresql
-    - wire-setup
+  tags: [postgresql, wire-setup]
+  # Requires: wire_pass
 
 - name: Deploy cluster monitoring
   import_playbook: postgresql-playbooks/postgresql-monitoring.yml
-  tags:
-    - postgresql
-    - monitoring
+  tags: [postgresql, monitoring]

ansible/postgresql-playbooks/postgresql-deploy-primary.yml

@@ -9,6 +9,33 @@
     replica_node2: "{{ hostvars[(groups.get('postgresql_ro', []) | last) | default('postgresql3')]['ansible_default_ipv4']['address'] | default(hostvars[(groups.get('postgresql_ro', []) | last) | default('postgresql3')]['ansible_host'] | default((groups.get('postgresql_ro', []) | last) | default('postgresql3'))) }}"
     pg_service_name: "postgresql@{{ postgresql_version }}-main.service"
   tasks:
+    # ===== PREREQUISITE VALIDATION =====
+    - name: Validate required secrets are available
+      ansible.builtin.assert:
+        that:
+          - repmgr_password is defined
+          - repmgr_password | length > 0
+        fail_msg: |
+          ❌ PREREQUISITE FAILED: repmgr_password is not available!
+
+          This playbook requires the repmgr password to be set as an Ansible fact.
+
+          Solution:
+            Run the complete deployment pipeline:
+              ansible-playbook postgresql-deploy.yml
+
+            OR run with the postgresql tag to include secrets:
+              ansible-playbook postgresql-deploy.yml --tags postgresql
+
+            OR run secrets playbook first:
+              ansible-playbook postgresql-playbooks/postgresql-secrets.yml
+              ansible-playbook postgresql-playbooks/postgresql-deploy-primary.yml
+
+          The postgresql-secrets.yml playbook fetches/creates passwords from Kubernetes
+          and sets them as Ansible facts for use by deployment playbooks.
+        success_msg: "✅ Prerequisites validated: repmgr_password is available"
+      run_once: true
+
     - name: Ensure repmgr scripts directory exists
       ansible.builtin.file:
         path: /opt/repmgr/scripts
@@ -103,11 +130,18 @@
           when: repmgr_user_check.stdout.strip() == "0"
           register: create_repmgr_user
 
+        - name: Update repmgr user password (if user already exists)
+          ansible.builtin.shell: |
+            sudo -u postgres psql -c "ALTER USER {{ repmgr_user }} WITH PASSWORD '{{ repmgr_password }}';"
+          when: repmgr_user_check.stdout.strip() != "0"
+          register: update_repmgr_password
+          no_log: true
+
         - name: Display user creation result
           ansible.builtin.debug:
             msg: |
-              repmgr user status: {{ 'CREATED' if repmgr_user_check.stdout.strip() == "0" else 'ALREADY EXISTS' }}
-          when: create_repmgr_user is defined
+              repmgr user status: {{ 'CREATED' if repmgr_user_check.stdout.strip() == "0" else 'ALREADY EXISTS (password updated)' }}
+          when: create_repmgr_user is defined or update_repmgr_password is defined
 
         - name: Check if repmgr database exists
           ansible.builtin.shell: |

ansible/postgresql-playbooks/postgresql-deploy-replica.yml

@@ -9,6 +9,33 @@
     current_replica: "{{ ansible_default_ipv4.address | default(ansible_host) }}"
     pg_service_name: "postgresql@{{ postgresql_version }}-main.service"
   tasks:
+    # ===== PREREQUISITE VALIDATION =====
+    - name: Validate required secrets are available
+      ansible.builtin.assert:
+        that:
+          - repmgr_password is defined
+          - repmgr_password | length > 0
+        fail_msg: |
+          ❌ PREREQUISITE FAILED: repmgr_password is not available!
+
+          This playbook requires the repmgr password to be set as an Ansible fact.
+
+          Solution:
+            Run the complete deployment pipeline:
+              ansible-playbook postgresql-deploy.yml
+
+            OR run with the postgresql tag to include secrets:
+              ansible-playbook postgresql-deploy.yml --tags postgresql
+
+            OR run secrets playbook first:
+              ansible-playbook postgresql-playbooks/postgresql-secrets.yml
+              ansible-playbook postgresql-playbooks/postgresql-deploy-replica.yml
+
+          The postgresql-secrets.yml playbook fetches/creates passwords from Kubernetes
+          and sets them as Ansible facts for use by deployment playbooks.
+        success_msg: "✅ Prerequisites validated: repmgr_password is available"
+      run_once: true
+
     # ===== INITIAL STATUS CHECK =====
     - name: Check replica configuration status
       block:

ansible/postgresql-playbooks/postgresql-secrets.yml

@@ -0,0 +1,145 @@
+---
+- name: Manage PostgreSQL passwords in Kubernetes Secrets
+  hosts: postgresql
+  become: no
+  gather_facts: yes
+  vars:
+    # Secret configurations
+    pg_secrets:
+      - name: "{{ repmgr_secret_name }}"
+        namespace: "{{ repmgr_namespace }}"
+        username: "{{ repmgr_user }}"
+        database: "{{ repmgr_database }}"
+        password_var: "repmgr_password"
+        component: "repmgr"
+        description: "repmgr HA management"
+      - name: "{{ wire_pg_secret_name }}"
+        namespace: "{{ wire_namespace | default('default') }}"
+        username: "{{ wire_user }}"
+        database: "{{ wire_dbname }}"
+        password_var: "wire_pass"
+        component: "wire-server"
+        description: "wire-server application"
+
+  tasks:
+    # ===== KUBERNETES CONNECTIVITY VALIDATION =====
+    - name: Validate kubectl is accessible
+      ansible.builtin.shell: kubectl cluster-info
+      register: kubectl_check
+      delegate_to: localhost
+      run_once: true
+      failed_when: false
+      changed_when: false
+
+    - name: Fail if kubectl is not accessible
+      ansible.builtin.fail:
+        msg: |
+          ERROR: Cannot access Kubernetes cluster!
+
+          Please ensure:
+          1. kubectl is installed and accessible
+          2. KUBECONFIG is set: export KUBECONFIG=/path/to/kubeconfig
+          3. kubectl can connect: kubectl cluster-info
+
+          Current KUBECONFIG: {{ lookup('env', 'KUBECONFIG') | default('not set') }}
+          Error: {{ kubectl_check.stderr | default('unknown') }}
+      when: kubectl_check.rc != 0
+      delegate_to: localhost
+      run_once: true
+
+    # ===== SECRET MANAGEMENT FOR EACH CREDENTIAL =====
+    - name: Check if Kubernetes Secrets exist
+      ansible.builtin.shell: |
+        kubectl get secret {{ item.name }} -n {{ item.namespace }} -o jsonpath='{.data.password}' 2>/dev/null | base64 -d
+      register: k8s_secret_checks
+      delegate_to: localhost
+      run_once: true
+      failed_when: false
+      changed_when: false
+      loop: "{{ pg_secrets }}"
+      loop_control:
+        label: "{{ item.name }}"
+
+    - name: Process secret check results and set passwords
+      ansible.builtin.set_fact:
+        "{{ item.item.password_var }}": "{{ item.stdout }}"
+        "{{ item.item.password_var }}_source": "kubernetes-secret"
+      when:
+        - item.rc == 0
+        - item.stdout != ""
+      run_once: true
+      loop: "{{ k8s_secret_checks.results }}"
+      loop_control:
+        label: "{{ item.item.name }}"
+
+    - name: Generate random passwords for secrets that don't exist
+      ansible.builtin.set_fact:
+        "{{ item.item.password_var }}": "{{ lookup('password', '/dev/null', chars=['ascii_letters', 'digits'], length=32) }}"
+        "{{ item.item.password_var }}_generated": true
+      when:
+        - item.rc != 0 or item.stdout == ""
+      run_once: true
+      loop: "{{ k8s_secret_checks.results }}"
+      loop_control:
+        label: "{{ item.item.name }}"
+      no_log: true  # Don't log generated passwords
+
+    - name: Display password sources
+      ansible.builtin.debug:
+        msg: |
+          PostgreSQL Secrets Status:
+          {% for secret in pg_secrets %}
+          - {{ secret.description }}:
+              Secret: {{ secret.name }}
+              Namespace: {{ secret.namespace }}
+              Status: {% if hostvars[groups['postgresql'][0]][secret.password_var + '_source'] is defined %}Retrieved from existing secret{% else %}Generated new password{% endif %}
+          {% endfor %}
+      run_once: true
+
+    - name: Create or update Kubernetes Secrets
+      ansible.builtin.shell: |
+        kubectl create secret generic {{ item.item.name }} \
+          --namespace={{ item.item.namespace }} \
+          --from-literal=password='{{ hostvars[groups['postgresql'][0]][item.item.password_var] }}' \
+          --from-literal=username='{{ item.item.username }}' \
+          --from-literal=database='{{ item.item.database }}' \
+          --dry-run=client -o yaml | \
+        kubectl label --local -f - \
+          app=postgresql \
+          component={{ item.item.component }} \
+          managed-by=ansible \
+          --dry-run=client -o yaml | \
+        kubectl apply -f -
+      delegate_to: localhost
+      run_once: true
+      when:
+        - hostvars[groups['postgresql'][0]][item.item.password_var + '_generated'] is defined
+        - hostvars[groups['postgresql'][0]][item.item.password_var + '_generated']
+      loop: "{{ k8s_secret_checks.results }}"
+      loop_control:
+        label: "{{ item.item.name }}"
+
+    # ===== PROPAGATE FACTS TO ALL POSTGRESQL HOSTS =====
+    - name: Ensure password facts are available on all postgresql hosts
+      ansible.builtin.set_fact:
+        "{{ item.password_var }}": "{{ hostvars[groups['postgresql'][0]][item.password_var] }}"
+      loop: "{{ pg_secrets }}"
+      loop_control:
+        label: "{{ item.password_var }}"
+
+    - name: Display secret management completion
+      ansible.builtin.debug:
+        msg: |
+          ===== POSTGRESQL SECRETS SETUP COMPLETE =====
+          Host: {{ inventory_hostname }}
+
+          Secrets configured:
+          {% for secret in pg_secrets %}
+          - {{ secret.description }}:
+              Secret: {{ secret.name }}
+              Namespace: {{ secret.namespace }}
+              Access: kubectl get secret {{ secret.name }} -n {{ secret.namespace }} -o jsonpath='{.data.password}' | base64 --decode
+          {% endfor %}
+
+          All passwords are now available as Ansible facts for subsequent playbooks.
+      run_once: true