Skip to content

Commit

Permalink
Merge pull request #1729 from GhostWriters/variable-naming
Browse files Browse the repository at this point in the history
feat: 💥 new environment and volume var naming
  • Loading branch information
CLHatch authored Sep 7, 2024
2 parents 8a4b330 + dbe237a commit 370cc61
Show file tree
Hide file tree
Showing 631 changed files with 2,736 additions and 3,067 deletions.
8 changes: 4 additions & 4 deletions .github/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,14 @@ Look at the **App Specifics** list in the **nav** section and add a new line for
- `com.dockstarter.appinfo.nicename: "<AppName>"` must match `<appname>` exactly but can have mixed case. Ex: Portainer vs PORTAINER
- `com.dockstarter.appvars.<appname>_enabled: "false"` must be included and default to false. Users pick which apps are enabled
- `com.dockstarter.appvars.<appname>_network_mode: ""` must be included and default to blank.
- `com.dockstarter.appvars.<appname>_<var_name>: "<var_value>"` one entry for each variable specific to the app environment. See existing apps for examples
- `com.dockstarter.appvars.<appname>_environment_<var_name>: "<var_value>"` one entry for each variable specific to the app environment. See existing apps for examples
- `logging` and the items beneath it should be included exactly as shown in other apps
- `restart` should be `unless-stopped` or should include a comment about why another option is used
- `volumes` should contain the volumes used by the app
- `- /etc/localtime:/etc/localtime:ro` is always included
- `- ${DOCKERCONFDIR}/<appname>:<container_config>` should be used to define the primary config directory for the app
- `- ${DOCKERSTORAGEDIR}:/storage` is always included
- `<appname>.hostname.yml` sets the hostname to use the `${DOCKERHOSTNAME}` variable
- `- ${DOCKER_VOLUME_CONFIG}/<appname>:<container_config>` should be used to define the primary config directory for the app
- `- ${DOCKER_VOLUME_STORAGE}:/storage` is always included
- `<appname>.hostname.yml` sets the hostname to use the `${DOCKER_HOSTNAME}` variable
- `<appname>.netmode.yml` contains the `<APPNAME>_NETWORK_MODE` variable
- `<appname>.ports.yml` contains the ports used by the app. This file can be excluded if the app does not require ports
- At least one of the following files must be included:
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ name: Build mkdocs and deploy to GitHub Pages
on: [push, pull_request]

jobs:

build:
name: Build docs
runs-on: ubuntu-latest
Expand Down
70 changes: 60 additions & 10 deletions .scripts/appvars_create.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,74 @@ appvars_create() {
APPNAME=${APPNAME^^}
local FILENAME=${APPNAME,,}
local APPTEMPLATES="${SCRIPTPATH}/compose/.apps/${FILENAME}"
local APPLABELFILE="${APPTEMPLATES}/${FILENAME}.labels.yml"

local -A APP_VAR_VALUE
local -A APP_VAR_MIGRATE

# Build variable values lookup array, APP_VAR_VALUES["variable"]="default value"
{
# Read all lines with labels into temporary APP_LABEL_LINES array
local -a APP_LABEL_LINES
readarray -t APP_LABEL_LINES < <(grep --color=never -P "\scom\.dockstarter\.appvars\.\K[\w]+" "${APPLABELFILE}" || true)
if [[ -z ${APP_LABEL_LINES[*]} ]]; then
error "Unable to find labels for ${APPNAME}"
return
fi

for line in "${APP_LABEL_LINES[@]}"; do
local SET_VAR
local SET_VAL
SET_VAR=$(echo "$line" | grep --color=never -Po "\scom\.dockstarter\.appvars\.\K[\w]+")
SET_VAL=$(echo "$line" | grep --color=never -Po "\scom\.dockstarter\.appvars\.${SET_VAR}: \K.*" | sed -E 's/^([^"].*[^"])$/"\1"/' | xargs || true)
if [[ -n ${SET_VAR} ]]; then
APP_VAR_VALUE["${SET_VAR^^}"]=${SET_VAL}
fi
done
}

# Build migrate variable lookup array, APP_MIGRATE_VAR["variable"]="migrate from variable"
for SET_VAR in "${!APP_VAR_VALUE[@]}"; do
local APPNAME=${SET_VAR%%_*}
local REST_VAR=${SET_VAR#"${APPNAME}_"}
local VAR_TYPE=${REST_VAR%%_*}
case "${VAR_TYPE}" in
ENVIRONMENT | VOLUME)
REST_VAR=${REST_VAR#"${VAR_TYPE}"}
local MIGRATE_VAR="${APPNAME}${REST_VAR}"
# shellcheck disable=SC2199
if [[ " ${!APP_VAR_VALUE[@]} " != *" ${MIGRATE_VAR} "* ]]; then
# Potential "migrate from" variable isn't an existing app variable, add it to the migrate list
APP_VAR_MIGRATE["${SET_VAR}"]=${MIGRATE_VAR}
fi
;;
esac
done

# Actual processing starts here
info "Creating environment variables for ${APPNAME}."
while IFS= read -r line; do
local VAR_LABEL=${line}
local SET_VAR=${VAR_LABEL^^}
if grep -q -P "^${SET_VAR}=" "${COMPOSE_ENV}"; then
for SET_VAR in "${!APP_VAR_VALUE[@]}"; do
if grep -q -P "^\s*${SET_VAR}\s*=" "${COMPOSE_ENV}"; then
# Variable already exists
continue
fi

local DEFAULT_VAL
DEFAULT_VAL=$(grep --color=never -Po "\scom\.dockstarter\.appvars\.${VAR_LABEL}: \K.*" "${APPTEMPLATES}/${FILENAME}.labels.yml" | sed -E 's/^([^"].*[^"])$/"\1"/' | xargs || true)
echo "${SET_VAR}=" >> "${COMPOSE_ENV}"
run_script 'env_set' "${SET_VAR}" "${DEFAULT_VAL}"
done < <(grep --color=never -Po "\scom\.dockstarter\.appvars\.\K[\w]+" "${APPTEMPLATES}/${FILENAME}.labels.yml" || error "Unable to find labels for ${APPNAME}")
local MIGRATE_VAR=${APP_VAR_MIGRATE["${SET_VAR}"]-}
if [[ -n ${MIGRATE_VAR} ]] && grep -q -P "^\s*${MIGRATE_VAR}\s*=" "${COMPOSE_ENV}"; then
# Migrate old variable
run_script 'env_rename' "${MIGRATE_VAR}" "${SET_VAR}"
else
# Add new variable
local DEFAULT_VAL=${APP_VAR_VALUE["${SET_VAR}"]}
notice "Adding ${SET_VAR}='${DEFAULT_VAL}' in ${COMPOSE_ENV} file."
run_script 'env_set' "${SET_VAR}" "${DEFAULT_VAL}"
fi
done
run_script 'env_set' "${APPNAME}_ENABLED" true
}

test_appvars_create() {
run_script 'env_update'
run_script 'appvars_create' WATCHTOWER
run_script 'env_update'
cat "${COMPOSE_ENV}"
}
2 changes: 1 addition & 1 deletion .scripts/appvars_create_all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ appvars_create_all() {
}

test_appvars_create_all() {
run_script 'env_update'
run_script 'appvars_create_all'
run_script 'env_update'
cat "${COMPOSE_ENV}"
}
2 changes: 1 addition & 1 deletion .scripts/appvars_purge.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ appvars_purge() {
}

test_appvars_purge() {
run_script 'env_update'
run_script 'appvars_purge' WATCHTOWER
run_script 'env_update'
cat "${COMPOSE_ENV}"
}
2 changes: 1 addition & 1 deletion .scripts/appvars_purge_all.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ appvars_purge_all() {
}

test_appvars_purge_all() {
run_script 'env_update'
run_script 'appvars_purge_all'
run_script 'env_update'
cat "${COMPOSE_ENV}"
}
29 changes: 29 additions & 0 deletions .scripts/appvars_rename.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -Eeuo pipefail
IFS=$'\n\t'

appvars_rename() {
local FROMAPP=${1-}
local TOAPP=${2-}
local FROMAPP_ENABLED
FROMAPP_ENABLED=$(run_script 'env_get' "${FROMAPP^^}_ENABLED")
local TOAPP_ENABLED
TOAPP_ENABLED=$(run_script 'env_get' "${TOAPP^^}_ENABLED")
if [[ ${FROMAPP_ENABLED} == true ]] && [[ ${TOAPP_ENABLED} != true ]]; then
notice "Migrating from ${FROMAPP^^} to ${TOAPP^^}."
docker stop "${FROMAPP,,}" || warn "Failed to stop ${FROMAPP,,} container.\nFailing command: ${F[C]}docker stop ${FROMAPP,,}"
notice "Moving config folder."
local DOCKER_VOLUME_CONFIG
DOCKER_VOLUME_CONFIG=$(run_script 'env_get' DOCKER_VOLUME_CONFIG)
mv "${DOCKER_VOLUME_CONFIG}/${FROMAPP,,}" "${DOCKER_VOLUME_CONFIG}/${TOAPP,,}" || warn "Failed to move folder.\nFailing command: ${F[C]}mv \"${DOCKER_VOLUME_CONFIG}/${FROMAPP,,}\" \"${DOCKER_VOLUME_CONFIG}/${TOAPP,,}\""
notice "Migrating vars."
sed -i "s/^\s*${FROMAPP^^}_/${TOAPP^^}_/" "${COMPOSE_ENV}" || fatal "Failed to migrate vars from ${FROMAPP^^}_ to ${TOAPP^^}_\nFailing command: ${F[C]}sed -i \"s/^\\s*${FROMAPP^^}_/${TOAPP^^}_/\" \"${COMPOSE_ENV}\""
run_script 'appvars_create' "${TOAPP^^}"
notice "Completed migrating from ${FROMAPP^^} to ${TOAPP^^}. Run ${F[C]}ds -c${NC} to create the new container."
fi
}

test_appvars_rename() {
# run_script 'appvars_rename'
warn "CI does not test appvars_rename."
}
2 changes: 1 addition & 1 deletion .scripts/config_global.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ IFS=$'\n\t'

config_global() {
local APPNAME="Global"
local VARNAMES=(DOCKERCONFDIR DOCKERSTORAGEDIR DOCKERHOSTNAME PGID PUID TZ)
local VARNAMES=(DOCKER_VOLUME_CONFIG DOCKER_VOLUME_STORAGE DOCKER_HOSTNAME PGID PUID TZ)
local APPVARS
APPVARS=$(for v in "${VARNAMES[@]}"; do echo "${v}=$(run_script 'env_get' "${v}")"; done)

Expand Down
35 changes: 0 additions & 35 deletions .scripts/config_vpn.sh

This file was deleted.

15 changes: 0 additions & 15 deletions .scripts/detect_lan_network.sh

This file was deleted.

22 changes: 11 additions & 11 deletions .scripts/env_backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,23 @@ IFS=$'\n\t'

env_backup() {
run_script 'env_create'
local DOCKERCONFDIR
DOCKERCONFDIR=$(run_script 'env_get' DOCKERCONFDIR)
info "Taking ownership of ${DOCKERCONFDIR} (non-recursive)."
sudo chown "${DETECTED_PUID}":"${DETECTED_PGID}" "${DOCKERCONFDIR}" > /dev/null 2>&1 || true
local DOCKER_VOLUME_CONFIG
DOCKER_VOLUME_CONFIG=$(run_script 'env_get' DOCKER_VOLUME_CONFIG)
info "Taking ownership of ${DOCKER_VOLUME_CONFIG} (non-recursive)."
sudo chown "${DETECTED_PUID}":"${DETECTED_PGID}" "${DOCKER_VOLUME_CONFIG}" > /dev/null 2>&1 || true
local BACKUPTIME
BACKUPTIME=$(date +"%Y%m%d%H%M%S")
info "Copying .env file to ${DOCKERCONFDIR}/.compose.backups/.env.${BACKUPTIME}"
mkdir -p "${DOCKERCONFDIR}/.compose.backups" || fatal "Failed to make directory.\nFailing command: ${F[C]}mkdir -p \"${DOCKERCONFDIR}/.compose.backups\""
cp "${COMPOSE_ENV}" "${DOCKERCONFDIR}/.compose.backups/.env.${BACKUPTIME}" || fatal "Failed to copy backup.\nFailing command: ${F[C]}cp \"${COMPOSE_ENV}\" \"${DOCKERCONFDIR}/.compose.backups/.env.${BACKUPTIME}\""
run_script 'set_permissions' "${DOCKERCONFDIR}/.compose.backups"
info "Copying .env file to ${DOCKER_VOLUME_CONFIG}/.compose.backups/.env.${BACKUPTIME}"
mkdir -p "${DOCKER_VOLUME_CONFIG}/.compose.backups" || fatal "Failed to make directory.\nFailing command: ${F[C]}mkdir -p \"${DOCKER_VOLUME_CONFIG}/.compose.backups\""
cp "${COMPOSE_ENV}" "${DOCKER_VOLUME_CONFIG}/.compose.backups/.env.${BACKUPTIME}" || fatal "Failed to copy backup.\nFailing command: ${F[C]}cp \"${COMPOSE_ENV}\" \"${DOCKER_VOLUME_CONFIG}/.compose.backups/.env.${BACKUPTIME}\""
run_script 'set_permissions' "${DOCKER_VOLUME_CONFIG}/.compose.backups"
info "Removing old .env backups."
find "${DOCKERCONFDIR}/.compose.backups" -type f -name ".env.*" -mtime +3 -delete > /dev/null 2>&1 || warn "Old .env backups not removed."
find "${DOCKER_VOLUME_CONFIG}/.compose.backups" -type f -name ".env.*" -mtime +3 -delete > /dev/null 2>&1 || warn "Old .env backups not removed."

# Backup location has moved
if [[ -d "${DOCKERCONFDIR}/.env.backups" ]]; then
if [[ -d "${DOCKER_VOLUME_CONFIG}/.env.backups" ]]; then
info "Removing old backup location."
rm -rf "${DOCKERCONFDIR}/.env.backups" || fatal "Failed to remove directory.\nFailing command: ${F[C]}rm -rf \"${DOCKERCONFDIR}/.env.backups\""
rm -rf "${DOCKER_VOLUME_CONFIG}/.env.backups" || fatal "Failed to remove directory.\nFailing command: ${F[C]}rm -rf \"${DOCKER_VOLUME_CONFIG}/.env.backups\""
fi
}

Expand Down
4 changes: 2 additions & 2 deletions .scripts/env_get.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ IFS=$'\n\t'
env_get() {
local GET_VAR=${1-}
local VAR_FILE=${2:-$COMPOSE_ENV}
grep --color=never -Po "^${GET_VAR}=\K.*" "${VAR_FILE}" | xargs || true
grep --color=never -Po "^\s*${GET_VAR}\s*=\K.*" "${VAR_FILE}" | tail -1 | xargs || true
}

test_env_get() {
run_script 'env_get' DOCKERCONFDIR
run_script 'env_get' DOCKER_VOLUME_CONFIG
}
17 changes: 17 additions & 0 deletions .scripts/env_rename.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -Eeuo pipefail
IFS=$'\n\t'

env_rename() {
local FROMVAR=${1-}
local TOVAR=${2-}
if grep -q -P "^\s*${FROMVAR}\s*=" "${COMPOSE_ENV}"; then
notice "Renaming ${FROMVAR} to ${TOVAR} in ${COMPOSE_ENV} file."
sed -i "s/^\s*${FROMVAR}\s*=/${TOVAR}=/" "${COMPOSE_ENV}" || fatal "Failed to rename var from ${FROMVAR} to ${TOVAR}\nFailing command: ${F[C]}sed -i \"s/^\\s*${FROMVAR}\\s*=/${TOVAR}=/\" \"${COMPOSE_ENV}\""
fi
}

test_env_rename() {
# run_script 'env_rename'
warn "CI does not test env_rename."
}
65 changes: 16 additions & 49 deletions .scripts/env_sanitize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,67 +3,34 @@ set -Eeuo pipefail
IFS=$'\n\t'

env_sanitize() {
# Replace ~ with /home/username
if grep -q -P '^\w+DIR=~/' "${COMPOSE_ENV}"; then
info "Replacing ~ with ${DETECTED_HOMEDIR} in ${COMPOSE_ENV} file."
sed -i -E "s/^(\w+DIR)=~\//\1=$(sed 's/[&/\]/\\&/g' <<< "${DETECTED_HOMEDIR}")\//g" "${COMPOSE_ENV}" | warn "Please verify that ~ is not used in ${COMPOSE_ENV} file."
fi

# Set LAN_NETWORK using detect_lan_network
local LAN_NETWORK
LAN_NETWORK=$(run_script 'env_get' LAN_NETWORK)
if grep -q -P 'x' <<< "${LAN_NETWORK}" || [[ ${LAN_NETWORK} == "" ]]; then
local DETECTED_LAN_NETWORK
DETECTED_LAN_NETWORK=$(run_script 'detect_lan_network')
run_script 'env_set' LAN_NETWORK "${DETECTED_LAN_NETWORK}"
fi

# Don't run Ouroboros and Watchtower at the same time
local OUROBOROS_ENABLED
OUROBOROS_ENABLED=$(run_script 'env_get' OUROBOROS_ENABLED)
local WATCHTOWER_ENABLED
WATCHTOWER_ENABLED=$(run_script 'env_get' WATCHTOWER_ENABLED)
if [[ ${OUROBOROS_ENABLED} == true ]] && [[ ${WATCHTOWER_ENABLED} == true ]]; then
run_script 'env_set' OUROBOROS_ENABLED false
fi

# Don't set OUROBOROS_NETWORK_MODE to none
local OUROBOROS_NETWORK_MODE
OUROBOROS_NETWORK_MODE=$(run_script 'env_get' OUROBOROS_NETWORK_MODE)
if [[ ${OUROBOROS_NETWORK_MODE} == "none" ]]; then
run_script 'env_set' OUROBOROS_NETWORK_MODE ""
fi

# Don't set WATCHTOWER_NETWORK_MODE to none
local WATCHTOWER_NETWORK_MODE
WATCHTOWER_NETWORK_MODE=$(run_script 'env_get' WATCHTOWER_NETWORK_MODE)
if [[ ${WATCHTOWER_NETWORK_MODE} == "none" ]]; then
run_script 'env_set' WATCHTOWER_NETWORK_MODE ""
fi

# Migrate from LetsEncrypt to SWAG
local LETSENCRYPT_ENABLED
LETSENCRYPT_ENABLED=$(run_script 'env_get' LETSENCRYPT_ENABLED)
local SWAG_ENABLED
SWAG_ENABLED=$(run_script 'env_get' SWAG_ENABLED)
if [[ ${LETSENCRYPT_ENABLED} == true ]] && [[ ${SWAG_ENABLED} != true ]]; then
notice "Migrating from LETSENCRYPT to SWAG."
docker stop letsencrypt || warn "Failed to stop letsencrypt container.\nFailing command: ${F[C]}docker stop letsencrypt"
notice "Moving config folder."
local DOCKERCONFDIR
DOCKERCONFDIR=$(run_script 'env_get' DOCKERCONFDIR)
mv "${DOCKERCONFDIR}/letsencrypt" "${DOCKERCONFDIR}/swag" || fatal "Failed to move folder.\nFailing command: ${F[C]}mv \"${DOCKERCONFDIR}/letsencrypt\" \"${DOCKERCONFDIR}/swag\""
notice "Migrating vars."
sed -i "s/^LETSENCRYPT_/SWAG_/" "${COMPOSE_ENV}" || fatal "Failed to migrate vars from LETSENCRYPT_ to SWAG_\nFailing command: ${F[C]}sed -i \"s/^LETSENCRYPT_/SWAG_/\" \"${COMPOSE_ENV}\""
run_script 'appvars_create' SWAG
notice "Completed migrating from LETSENCRYPT to SWAG. Run ${F[C]}ds -c${NC} to create the new container."
# Rename vars
run_script 'env_rename' DOCKERCONFDIR DOCKER_VOLUME_CONFIG
run_script 'env_rename' DOCKERGID DOCKER_GID
run_script 'env_rename' DOCKERHOSTNAME DOCKER_HOSTNAME
run_script 'env_rename' DOCKERSTORAGEDIR DOCKER_VOLUME_STORAGE

# Rename apps
run_script 'appvars_rename' LETSENCRYPT SWAG
run_script 'appvars_rename' MINECRAFT_BEDROCK_SERVER MINECRAFTBEDROCKSERVER
run_script 'appvars_rename' MINECRAFT_SERVER MINECRAFTSERVER

# Replace ~ with /home/username
if grep -q -P '^\w+_VOLUME_\w+=~/' "${COMPOSE_ENV}"; then
info "Replacing ~ with ${DETECTED_HOMEDIR} in ${COMPOSE_ENV} file."
sed -i -E "s/^(\w+_VOLUME_\w+)=~\//\1=$(sed 's/[&/\]/\\&/g' <<< "${DETECTED_HOMEDIR}")\//g" "${COMPOSE_ENV}" | warn "Please verify that ~ is not used in ${COMPOSE_ENV} file."
fi

}

test_env_sanitize() {
run_script 'appvars_create' OUROBOROS
run_script 'appvars_create' WATCHTOWER
run_script 'env_sanitize'
run_script 'appvars_purge' OUROBOROS
run_script 'appvars_purge' WATCHTOWER
}
11 changes: 3 additions & 8 deletions .scripts/env_set.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,9 @@ env_set() {
# https://unix.stackexchange.com/questions/422165/escape-double-quotes-in-variable/422170#422170
NEW_VAL=$(printf "%s\n" "${2-}" | sed -e "s/'/'\"'\"'/g" -e "1s/^/'/" -e "\$s/\$/'/")
local VAR_FILE=${3:-$COMPOSE_ENV}
local VAR_VAL
VAR_VAL=$(grep --color=never -P "^${SET_VAR}=" "${VAR_FILE}") || fatal "Failed to find ${SET_VAR} in ${VAR_FILE}\nFailing command: ${F[C]}grep --color=never -P \"^${SET_VAR}=\" \"${VAR_FILE}\""
# https://stackoverflow.com/questions/29613304/is-it-possible-to-escape-regex-metacharacters-reliably-with-sed/29613573#29613573
local SED_FIND
SED_FIND=$(sed 's/[^^]/[&]/g; s/\^/\\^/g' <<< "${VAR_VAL}")
local SED_REPLACE
SED_REPLACE=$(sed 's/[&/\]/\\&/g' <<< "${SET_VAR}=${NEW_VAL}")
sed -i "s/^${SED_FIND}$/${SED_REPLACE}/" "${VAR_FILE}" || fatal "Failed to set ${SED_REPLACE}\nFailing command: ${F[C]}sed -i \"s/^${SED_FIND}$/${SED_REPLACE}/\" \"${VAR_FILE}\""

sed -i "/^\s*${SET_VAR}\s*=/d" "${VAR_FILE}" || true
echo "${SET_VAR}=${NEW_VAL}" >> "${VAR_FILE}" || fatal "Failed to set ${SET_VAR}=${NEW_VAL}\nFailing command: ${F[C]} \"echo ${SET_VAR}=${NEW_VAL}\" >> \"${VAR_FILE}\""
}

test_env_set() {
Expand Down
Loading

0 comments on commit 370cc61

Please sign in to comment.