diff --git a/docs/infrastructure-ovn-setup.md b/docs/infrastructure-ovn-setup.md index 4ab974a9..5e097d89 100644 --- a/docs/infrastructure-ovn-setup.md +++ b/docs/infrastructure-ovn-setup.md @@ -123,11 +123,22 @@ If there's ever a need to reconfigure a node, simply remove the label and the Da !!! note - To upload backups to a Ceph Swift API gateway, edit ovn-backup.config to set - `SWIFT_UPLOAD' "true"`, edit the other related options appropriately (i.e., - set the SWIFT_BASE_URL and CONTAINER) and put the username and secret key of - the account to use in `swift-account.env` before running `kubectl apply` an - indicated above. + To upload backups to Swift with tempauth, edit + /opt/genestack/kustomize/ovn/ovn-backup/ovn-backup.config to set + `SWIFT_TEMPAUTH_UPLOAD' "true"`, edit the other related options + appropriately (i.e., set the CONTAINER) and fill the ST_AUTH, ST_USER, and + ST_KEY as appropriate for the Swift CLI client in the `swift-tempauth.env` + file and then run: + + kubectl apply -k /opt/genestack/kustomize/ovn/ovn-backup \ + --prune -l app=ovn-backup \ + --prune-allowlist=core/v1/Secret \ + --prune-allowlist=core/v1/ConfigMap + + If you need to change variables in the future, you can edit the relevant + files and use `kubectl` with these prune options to avoid accumulating + old ConfigMaps and Secrets from successive `kubectl apply` operations, but + you can omit the pruning options if desired. ## Centralize `kube-ovn-controller` pods diff --git a/kustomize/ovn/kustomization.yaml b/kustomize/ovn/kustomization.yaml index 0205b83f..84a68f18 100644 --- a/kustomize/ovn/kustomization.yaml +++ b/kustomize/ovn/kustomization.yaml @@ -1,17 +1,2 @@ -secretGenerator: - - name: ovn-backup-swift-account - namespace: kube-system - envs: - - swift-account.env -configMapGenerator: - - name: ovn-backup-script - namespace: kube-system - files: - - ovn-backup.sh - - name: ovn-backup-config - namespace: kube-system - envs: - - ovn-backup.config resources: - ovn-setup.yaml - - ovn-backup.yaml diff --git a/kustomize/ovn/ovn-backup.sh b/kustomize/ovn/ovn-backup.sh deleted file mode 100644 index 2bda515e..00000000 --- a/kustomize/ovn/ovn-backup.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash - -if [[ "$LOG_LEVEL" == "DEBUG" ]] -then - set -x -fi - -SWIFT_CONTAINER_BASE_URL="$SWIFT_BASE_URL/swift/v1" -export SWIFT_CONTAINER_BASE_URL - -log_level() { - local LEVEL="$1" - case "$LEVEL" in - DEBUG) - echo 5 - ;; - INFO) - echo 4 - ;; - WARNING) - echo 3 - ;; - ERROR) - echo 2 - ;; - CRITICAL) - echo 1 - ;; - *) - exit 3 - ;; - esac -} -export -f log_level - -log_line() { - local LEVEL - LEVEL="$(log_level "$1")" - if [[ "$LEVEL" -ge "$(log_level "$LOG_LEVEL")" ]] - then - local line - line=$(date +"%b %d %H:%M:%S $*") - echo "$line" | tee -a "$LOG_FILE" - fi -} -export -f log_line # exported for upload_file - -# Delete old backup files on volume. -cd "$BACKUP_DIR" || exit 2 -find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete; - -# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR -YMD="$(date +"%Y/%m/%d")" -mkdir -p "$YMD" && cd "$YMD" || exit 2 # kubectl-ko creates backups in $PWD, so we cd first. -/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed" -/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed" - -if [[ "$SWIFT_UPLOAD" != "true" ]] -then - exit 0 -fi - -# Everything from here forward deals with uploading to a Ceph Swift API gateway. - -cd "$BACKUP_DIR" || exit 2 -CURL="$(which curl)" -export CONTAINER CURL # these need to reach the subshell below used with `find` -HEADER_TEMP_FILE=$(mktemp /tmp/headers.XXXXXXXX) -$CURL -sS -D "$HEADER_TEMP_FILE" -H "X-Auth-User: $USERNAME" -H "X-Auth-Key: $SECRET_KEY" "$SWIFT_BASE_URL/auth/v1.1" -sed -i -e 's/\r//g' "$HEADER_TEMP_FILE" # strip carriage returns -token=$(awk '/X-Auth-Token/ { print $2 }' "$HEADER_TEMP_FILE") -rm "$HEADER_TEMP_FILE" -export token - -# wrap curl with some things we will always use -curl_wrap() { - $CURL -sS -H "X-Auth-Token: $token" "$@" -} -export -f curl_wrap - -# Create the container if it doesn't exist -check_container=$(curl_wrap -o /dev/null -w "%{http_code}" "$SWIFT_CONTAINER_BASE_URL/$CONTAINER") -if ! [[ "$check_container" =~ 20[0-9] ]] -then - curl_wrap -X PUT "$SWIFT_CONTAINER_BASE_URL/$CONTAINER" -fi - -# upload_file uploads $1 to the container -upload_file() { - FILE="$1" - local curl_return - curl_return=$(curl_wrap -w "%{http_code}" \ - -X PUT "${SWIFT_CONTAINER_BASE_URL}/${CONTAINER}/$FILE" -T "$FILE") - if [[ "$curl_return" == "201" ]] - then - log_line INFO "SUCCESSFUL UPLOAD $FILE" - else - log_line ERROR "FAILURE API returned $curl_return uploading $FILE (expected 201)" - fi -} -export -f upload_file - -# find created backups and upload them -cd "$BACKUP_DIR" || exit 2 -# unusual find syntax to use an exported function from the shell -find "$YMD" -type f -exec bash -c 'upload_file "$0"' {} \; diff --git a/kustomize/ovn/ovn-backup/kustomization.yaml b/kustomize/ovn/ovn-backup/kustomization.yaml new file mode 100644 index 00000000..1603f82b --- /dev/null +++ b/kustomize/ovn/ovn-backup/kustomization.yaml @@ -0,0 +1,16 @@ +commonLabels: + app: ovn-backup +namespace: kube-system +secretGenerator: + - name: ovn-backup-swift-tempauth-account + envs: + - swift-tempauth.env +configMapGenerator: + - name: ovn-backup-script + files: + - ovn-backup.sh + - name: ovn-backup-config + envs: + - ovn-backup.config +resources: + - ovn-backup.yaml diff --git a/kustomize/ovn/ovn-backup.config b/kustomize/ovn/ovn-backup/ovn-backup.config similarity index 59% rename from kustomize/ovn/ovn-backup.config rename to kustomize/ovn/ovn-backup/ovn-backup.config index 0c37a232..d377c253 100644 --- a/kustomize/ovn/ovn-backup.config +++ b/kustomize/ovn/ovn-backup/ovn-backup.config @@ -6,11 +6,7 @@ BACKUP_DIR=/backup LOG_FILE=/backup/upload.log LOG_LEVEL=INFO -# From here forward, variables for uploading to a Ceph Swift interface. -SWIFT_UPLOAD=false -SWIFT_BASE_URL=http://FIX_ME:8081 - -# Nothing after this line makes any difference unless you used -# SWIFT_UPLOAD: "true" -# above. +# From here forward, variables for uploading to Swift with tempauth +SWIFT_TEMPAUTH_UPLOAD=false +# If you change this to "true", set the variables in swift-tempauth.env CONTAINER=test-ovn-backup diff --git a/kustomize/ovn/ovn-backup/ovn-backup.sh b/kustomize/ovn/ovn-backup/ovn-backup.sh new file mode 100644 index 00000000..a3c25083 --- /dev/null +++ b/kustomize/ovn/ovn-backup/ovn-backup.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +if [[ "$LOG_LEVEL" == "DEBUG" ]] +then + set -x +fi + +log_level() { + local LEVEL="$1" + case "$LEVEL" in + DEBUG) + echo 5 + ;; + INFO) + echo 4 + ;; + WARNING) + echo 3 + ;; + ERROR) + echo 2 + ;; + CRITICAL) + echo 1 + ;; + *) + exit 3 + ;; + esac +} +export -f log_level + +log_line() { + local LEVEL + LEVEL="$(log_level "$1")" + if [[ "$LEVEL" -le "$(log_level "$LOG_LEVEL")" ]] + then + local line + line=$(date +"%b %d %H:%M:%S $*") + echo "$line" | tee -a "$LOG_FILE" + fi +} +export -f log_line # exported for upload_file + +# Delete old backup files on volume. +cd "$BACKUP_DIR" || exit 2 +[[ -e "$BACKUP_DIR/last_upload" ]] || touch "$BACKUP_DIR/last_upload" || exit 3 +find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete; + +# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR +YMD="$(date +"%Y/%m/%d")" +# kubectl-ko creates backups in $PWD, so we cd first. +mkdir -p "$YMD" && cd "$YMD" || exit 2 +/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed" +/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed" + +if [[ "$SWIFT_TEMPAUTH_UPLOAD" != "true" ]] +then + exit 0 +fi + +# Everything from here forward deals with uploading to a Swift with tempauth. + +cd "$BACKUP_DIR" || exit 2 + +# Make a working "swift" command +SWIFT="kubectl -n openstack exec -i openstack-admin-client -- +env -i ST_AUTH=$ST_AUTH ST_USER=$ST_USER ST_KEY=$ST_KEY +/var/lib/openstack/bin/swift" +export SWIFT + +# Create the container if it doesn't exist +if ! $SWIFT stat "$CONTAINER" > /dev/null +then + $SWIFT post "$CONTAINER" +fi + +# upload_file uploads $1 to the container +upload_file() { + FILE="$1" + # Using OBJECT_NAME instead of FILE every time doesn't change the behavior, + # but stops shellcheck from identifying this as trying to read and write + # the same file. + OBJECT_NAME="$FILE" + if $SWIFT upload "$CONTAINER" --object-name "$OBJECT_NAME" - < "$FILE" + then + log_line INFO "SUCCESSFUL UPLOAD $FILE as object $OBJECT_NAME" + else + log_line ERROR "FAILURE API swift exited $? uploading $FILE as $OBJECT_NAME" + fi +} +export -f upload_file + +# find created backups and upload them +cd "$BACKUP_DIR" || exit 2 +# unusual find syntax to use an exported function from the shell +find "$YMD" -type f -newer "$BACKUP_DIR/last_upload" \ +-exec bash -c 'upload_file "$0"' {} \; +touch "$BACKUP_DIR/last_upload" diff --git a/kustomize/ovn/ovn-backup.yaml b/kustomize/ovn/ovn-backup/ovn-backup.yaml similarity index 94% rename from kustomize/ovn/ovn-backup.yaml rename to kustomize/ovn/ovn-backup/ovn-backup.yaml index 3fc1e6f9..edeafe63 100644 --- a/kustomize/ovn/ovn-backup.yaml +++ b/kustomize/ovn/ovn-backup/ovn-backup.yaml @@ -9,7 +9,6 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - namespace: kube-system name: ovndb-backup spec: accessModes: @@ -23,7 +22,6 @@ apiVersion: batch/v1 kind: CronJob metadata: name: ovn-snapshot-cron - namespace: kube-system spec: schedule: "0 0 * * *" concurrencyPolicy: Forbid @@ -50,7 +48,7 @@ spec: - configMapRef: name: ovn-backup-config - secretRef: - name: ovn-backup-swift-account + name: ovn-backup-swift-tempauth-account command: ["/backup-script/ovn-backup.sh"] image: docker.io/kubeovn/kube-ovn:v1.11.5 imagePullPolicy: IfNotPresent diff --git a/kustomize/ovn/ovn-backup/swift-tempauth.env b/kustomize/ovn/ovn-backup/swift-tempauth.env new file mode 100644 index 00000000..39f9231d --- /dev/null +++ b/kustomize/ovn/ovn-backup/swift-tempauth.env @@ -0,0 +1,6 @@ +# Set variables as appropriate for the Swift client for tempauth. +# ST_AUTH for the auth URL, +# e.g., https://tempauth.environment.yourdomain.invalid/auth/v1.0 +ST_AUTH=url +ST_USER=username +ST_KEY=passwordOrKey diff --git a/kustomize/ovn/swift-account.env b/kustomize/ovn/swift-account.env deleted file mode 100644 index d46f0326..00000000 --- a/kustomize/ovn/swift-account.env +++ /dev/null @@ -1,2 +0,0 @@ -USERNAME=username -SECRET_KEY=secret_key