feat: improve gitops workflow

Author: escalate

.github/workflows/test.yml

@@ -26,7 +26,8 @@ jobs:
 
       - name: Run build tool for ${{ matrix.arch }}-bit disk image
         run: |
-          export ANSIBLE_HOSTNAME='example.fritz.box'
-          export ANSIBLE_REPOSITORY_URL='https://github.com/escalate/ansible-gitops-example-repository.git'
+          export ANSIBLE_HOSTNAME=example.fritz.box
+          export ANSIBLE_HOSTGROUP=example
+          export ANSIBLE_REPOSITORY_URL=https://github.com/escalate/ansible-gitops-example-repository.git
           export ANSIBLE_VAULT_PASSWORD='REDACTED'
           make build-${{ matrix.arch }}bit

Makefile

@@ -4,13 +4,15 @@ SHELL = /bin/bash
 .PHONY: build-32bit
 build-32bit:
 	test -n "$(ANSIBLE_HOSTNAME)" # check env variable $$ANSIBLE_HOSTNAME
+	test -n "$(ANSIBLE_HOSTGROUP)" # check env variable $$ANSIBLE_HOSTGROUP
 	test -n "$(ANSIBLE_REPOSITORY_URL)" # check env variable $$ANSIBLE_REPOSITORY_URL
 	test -n "$(ANSIBLE_VAULT_PASSWORD)" # check env variable $$ANSIBLE_VAULT_PASSWORD
 	./build.sh 32
 
 .PHONY: build-64bit
 build-64bit:
 	test -n "$(ANSIBLE_HOSTNAME)" # check env variable $$ANSIBLE_HOSTNAME
+	test -n "$(ANSIBLE_HOSTGROUP)" # check env variable $$ANSIBLE_HOSTGROUP
 	test -n "$(ANSIBLE_REPOSITORY_URL)" # check env variable $$ANSIBLE_REPOSITORY_URL
 	test -n "$(ANSIBLE_VAULT_PASSWORD)" # check env variable $$ANSIBLE_VAULT_PASSWORD
 	./build.sh 64

README.md

@@ -2,29 +2,37 @@
 
 # Ansible GitOps - Raspberry Pi OS custom disk image
 
-A build tool based on [CustoPiZe](https://github.com/OctoPrint/CustoPiZer) to create a customized Raspberry Pi OS disk image which initiates the Ansible GitOps workflow on first boot.
+A build tool based on [CustoPiZe](https://github.com/OctoPrint/CustoPiZer) to create a custom Raspberry Pi OS disk image that starts the Ansible GitOps workflow on first boot.
 
 ## How does it work?
 
-The build tool downloads the latest [Raspberry Pi OS Lite (32-bit / 64-bit)](https://www.raspberrypi.com/software/operating-systems/) disk image and creates a systemd service inside it to bootstrap the Ansible GitOps workflow on first boot.
+The build tool downloads the latest [Raspberry Pi OS Lite (32-bit / 64-bit)](https://www.raspberrypi.com/software/operating-systems/) disk image and creates a systemd service in it that starts the Ansible GitOps workflow on first boot.
 
-After the Raspberry Pi boots successfully with the customized disk image, the systemd service prepares all needed dependencies and runs [ansible-pull](https://docs.ansible.com/ansible/latest/cli/ansible-pull.html) with the user pre-configured environment variables.
+After the Raspberry Pi has successfully booted with the customized disk image, the systemd service prepares all required [dependencies](https://github.com/escalate/ansible-gitops-raspberry-pi-os-custom-disk-image/blob/master/scripts/files/gitops-preparation.sh) and executes a [bootstrap script](https://github.com/escalate/ansible-gitops-raspberry-pi-os-custom-disk-image/blob/master/scripts/files/gitops-bootstrap.sh) with the settings preconfigured by the user.
 
-Ansible-pull checks out the pre-configured [Git repository](https://github.com/escalate/ansible-gitops-example-repository/) and runs the playbook `bootstrap.yml`.
-All steps for the further process must be stored in the `bootstrap.yml` playbook e.g. the preparation of an external USB drive as well as a cronjob for the periodical execution of ansible-pull.
+The [bootstrap script](https://github.com/escalate/ansible-gitops-raspberry-pi-os-custom-disk-image/blob/master/scripts/files/gitops-bootstrap.sh) checks out the preconfigured [Git repository](https://github.com/escalate/ansible-gitops-example-repository/), installs required roles and collections and runs the `bootstrap.yml` playbook.
+The `bootstrap.yml` playbook must contain all steps for the further process, e.g. the preparation of an external USB drive and a cronjob for the regular execution of the [deployment script](https://github.com/escalate/ansible-gitops-raspberry-pi-os-custom-disk-image/blob/master/scripts/files/gitops-deployment.sh).
 
-After the successful run of the playbook, a marker is set to prevent the systemd service from starting again at the next boot. Finally, the system is rebooted to complete all changes.
+After the successful run of the `bootstrap.yml` playbook, markers are set to prevent the scripts from being restarted at the next system start. Finally, the system is rebooted to complete all changes.
 
-Any configuration change of the Raspberry Pi should now be possible via the configured Git repository.
+All configuration changes on the Raspberry Pi should now be possible via the `site.yml` of the configured [Git repository](https://github.com/escalate/ansible-gitops-example-repository/).
 
 ## How to create a customized disk image?
 
-1. Define necessary environment variables needed for the later ansible-pull run.
+1. Define necessary environment variables needed for the later ansible-playbook run.
 
 ```
-export ANSIBLE_HOSTNAME='testserver.fritz.box'
-export ANSIBLE_REPOSITORY_URL='https://github.com/escalate/ansible-gitops-example-repository.git'
-export ANSIBLE_VAULT_PASSWORD='s3cret'
+# The Fully Qualified Domain Name (FQDN) of your server
+export ANSIBLE_HOSTNAME="testserver.fritz.box"
+"
+# The Ansible inventory group name where your server belongs to. For more information see https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html
+export ANSIBLE_HOSTGROUP="testing"
+
+# The URL of your Ansible control repository
+export ANSIBLE_REPOSITORY_URL"=https://github.com/escalate/ansible-gitops-example-repository.git"
+
+# The secret password to decrypt your Ansible Vault file. For more information see https://docs.ansible.com/ansible/latest/user_guide/vault.html
+export ANSIBLE_VAULT_PASSWORD="s3cret"
 ```
 
 2. Start the build process with one of the following commands.

build.sh

@@ -73,6 +73,7 @@ echo "Define customization environment variables"
 {
   echo "DOWNLOAD_IMAGE_ARCHIVE=${DOWNLOAD_IMAGE_ARCHIVE}"
   echo "ANSIBLE_HOSTNAME=${ANSIBLE_HOSTNAME}"
+  echo "ANSIBLE_HOSTGROUP=${ANSIBLE_HOSTGROUP}"
   echo "ANSIBLE_REPOSITORY_URL=${ANSIBLE_REPOSITORY_URL}"
   echo "ANSIBLE_VAULT_PASSWORD=${ANSIBLE_VAULT_PASSWORD}"
 } >"workspace/${ARCH}/gitops-config.env"

scripts/customize.sh

@@ -4,7 +4,7 @@ set -e
 
 export LC_ALL=C
 
-# shellcheck disable=SC1091
+# shellcheck source=/dev/null
 source /common.sh
 install_cleanup_trap
 
@@ -15,27 +15,30 @@ echo "Add default user pi with password raspberry"
 # shellcheck disable=SC2016
 echo 'pi:$6$MVEGdHTqed7c7za0$ESdhSXBIjSTVWKY7YWBII3UjQM6LhFur1alIXWJ9/Hf4mxgZqIuyX1yEsVf/qct4/sT0NStmvIPZs5de3SNNy0' >"${BOOT_PATH}/userconf.txt"
 
-echo "Copy original image archive"
+echo "Copy original disk image archive"
 cp "/files/${DOWNLOAD_IMAGE_ARCHIVE}" "/var/tmp/${DOWNLOAD_IMAGE_ARCHIVE}"
 cp "/files/${DOWNLOAD_IMAGE_ARCHIVE}.sha256" "/var/tmp/${DOWNLOAD_IMAGE_ARCHIVE}.sha256"
 
 echo "Set hostname"
 echo "${ANSIBLE_HOSTNAME}" >"/etc/hostname"
 sed --in-place "s/raspberrypi/${ANSIBLE_HOSTNAME}/g" "/etc/hosts"
 
+echo "Create GitOps scripts"
+cp "/files/gitops-bootstrap.sh" "/usr/local/bin/gitops-bootstrap.sh"
+cp "/files/gitops-deployment.sh" "/usr/local/bin/gitops-deployment.sh"
+cp "/files/gitops-preparation.sh" "/usr/local/bin/gitops-preparation.sh"
+cp "/files/gitops-utils.sh" "/usr/local/bin/gitops-utils.sh"
+
+echo "Define GitOps environment variables"
+tee "${BOOT_PATH}/gitops.env" <<EOT
+ANSIBLE_HOSTNAME="${ANSIBLE_HOSTNAME}"
+ANSIBLE_HOSTGROUP="${ANSIBLE_HOSTGROUP}"
+ANSIBLE_REPOSITORY_URL="${ANSIBLE_REPOSITORY_URL}"
+ANSIBLE_VAULT_PASSWORD="${ANSIBLE_VAULT_PASSWORD}"
+EOT
+
 echo "Create GitOps bootstrap systemd service"
 cp "/files/gitops-bootstrap.service" "/etc/systemd/system/gitops-bootstrap.service"
 
 echo "Enable GitOps bootstrap systemd service on startup"
 ln --symbolic "/etc/systemd/system/gitops-bootstrap.service" "/etc/systemd/system/multi-user.target.wants/gitops-bootstrap.service"
-
-echo "Create GitOps bootstrap scripts"
-cp "/files/gitops-install-and-configure-ansible.sh" "${BOOT_PATH}/gitops-install-and-configure-ansible.sh"
-cp "/files/gitops-run-ansible-playbook-and-reboot.sh" "${BOOT_PATH}/gitops-run-ansible-playbook-and-reboot.sh"
-
-echo "Define Ansible environment variables"
-tee "${BOOT_PATH}/gitops-bootstrap.env" <<EOT
-ANSIBLE_HOSTNAME=${ANSIBLE_HOSTNAME}
-ANSIBLE_REPOSITORY_URL=${ANSIBLE_REPOSITORY_URL}
-ANSIBLE_VAULT_PASSWORD=${ANSIBLE_VAULT_PASSWORD}
-EOT

scripts/files/gitops-bootstrap.service

@@ -1,14 +1,13 @@
 [Unit]
 Description=GitOps Bootstrap
 After=network-online.target
-ConditionPathExists=!/boot/firmware/gitops-bootstrap.done
 
 [Service]
 Type=oneshot
-EnvironmentFile=/boot/firmware/gitops-bootstrap.env
+EnvironmentFile=/boot/firmware/gitops.env
 ExecStartPre=/bin/sleep 60
-ExecStartPre=/boot/firmware/gitops-install-and-configure-ansible.sh
-ExecStart=/boot/firmware/gitops-run-ansible-playbook-and-reboot.sh
+ExecStartPre=/usr/local/bin/gitops-preparation.sh
+ExecStart=/usr/local/bin/gitops-bootstrap.sh
 RemainAfterExit=true
 
 [Install]

scripts/files/gitops-bootstrap.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e -E -u -C -o pipefail
+
+if [[ -f "/boot/firmware/gitops-bootstrap.done" ]]; then
+  echo "GitOps bootstrap already completed. Exiting."
+  exit 0
+fi
+
+# shellcheck source=/dev/null
+source "/boot/firmware/gitops.env"
+
+# shellcheck source=/dev/null
+source "/usr/local/bin/gitops-utils.sh"
+
+checkout_repository
+install_roles
+install_collections
+run_ansible_playbook "bootstrap.yml"
+
+echo "Create marker file"
+touch "/boot/firmware/gitops-bootstrap.done"
+
+echo "Reboot system"
+reboot

scripts/files/gitops-deployment.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e -E -u -C -o pipefail
+
+exec 1> >(logger --tag "$(basename "$0")") 2>&1
+
+# shellcheck source=/dev/null
+source "/boot/firmware/gitops.env"
+
+# shellcheck source=/dev/null
+source "/usr/local/bin/gitops-utils.sh"
+
+checkout_repository
+install_roles
+install_collections
+run_ansible_playbook "site.yml"

scripts/files/gitops-install-and-configure-ansible.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e -E -u -C -o pipefail
+
+if [[ -f "/boot/firmware/gitops-preparation.done" ]]; then
+  echo "GitOps preparation already completed. Exiting."
+  exit 0
+fi
+
+echo "Update package information"
+apt-get update
+
+echo "Install Python pip"
+apt-get --yes install python3-pip
+
+echo "Install Git"
+apt-get --yes install git
+
+echo "Install Ansible"
+pip3 install \
+  --disable-pip-version-check \
+  --break-system-packages \
+  --root-user-action=ignore \
+  ansible
+
+echo "Create Ansible configuration directory"
+mkdir --parent --verbose /etc/ansible
+
+echo "Create Ansible configuration file"
+tee "/etc/ansible/ansible.cfg" <<EOT
+[defaults]
+collections_path = /etc/ansible/collections
+interpreter_python = auto_silent
+nocows = true
+roles_path = /etc/ansible/roles
+vault_password_file = /etc/ansible/.vault_pass.txt
+verbosity = 1
+
+[diff]
+always = true
+EOT
+
+echo "Create Ansible inventory file"
+tee "/etc/ansible/hosts.yml" <<EOT
+${ANSIBLE_HOSTGROUP}:
+  hosts:
+    ${ANSIBLE_HOSTNAME}:
+      ansible_connection: local
+      ansible_host: 127.0.0.1
+EOT
+
+echo "Create Ansible Vault password file"
+echo "${ANSIBLE_VAULT_PASSWORD}" >/etc/ansible/.vault_pass.txt
+
+echo "Create marker file"
+touch "/boot/firmware/gitops-preparation.done"