From 43f0973e5172b6f919c8ab34d136c9e405116fb2 Mon Sep 17 00:00:00 2001
From: Adam Fabian <adam.fabian@rackspace.com>
Date: Fri, 14 Jun 2024 14:02:09 -0500
Subject: [PATCH] Switch to Swift tempauth for OVN backups (#304)

* Switch OVN backups to using Swift.

JIRA:OSPC-432

* Move OVN backup stuff to a separate directory.

This helps make it easy to use:

commonLabels:
  app: ovn-backup

to label all of the resources in kustomization.yaml, and the backup
functionality started cluttering up the main directory.

JIRA:OSPC-432

* For OVN backups, add script logic to upload new files since last upload.

We have MariaDB backups going every 6 hours now, and previous logic only
allowed for once-a-day backups.

JIRA:OSPC-432

* For OVN backups, make kustomize put the namespace for all resources.

JIRA:OSPC-432

* For OVN backups, switch Swift upload to Swift tempauth upload.

JIRA:OSPC-432
---
 docs/infrastructure-ovn-setup.md              |  21 +++-
 kustomize/ovn/kustomization.yaml              |  15 ---
 kustomize/ovn/ovn-backup.sh                   | 106 ------------------
 kustomize/ovn/ovn-backup/kustomization.yaml   |  16 +++
 .../ovn/{ => ovn-backup}/ovn-backup.config    |  10 +-
 kustomize/ovn/ovn-backup/ovn-backup.sh        |  99 ++++++++++++++++
 .../ovn/{ => ovn-backup}/ovn-backup.yaml      |   4 +-
 kustomize/ovn/ovn-backup/swift-tempauth.env   |   6 +
 kustomize/ovn/swift-account.env               |   2 -
 9 files changed, 141 insertions(+), 138 deletions(-)
 delete mode 100644 kustomize/ovn/ovn-backup.sh
 create mode 100644 kustomize/ovn/ovn-backup/kustomization.yaml
 rename kustomize/ovn/{ => ovn-backup}/ovn-backup.config (59%)
 create mode 100644 kustomize/ovn/ovn-backup/ovn-backup.sh
 rename kustomize/ovn/{ => ovn-backup}/ovn-backup.yaml (94%)
 create mode 100644 kustomize/ovn/ovn-backup/swift-tempauth.env
 delete mode 100644 kustomize/ovn/swift-account.env

diff --git a/docs/infrastructure-ovn-setup.md b/docs/infrastructure-ovn-setup.md
index 4ab974a9..5e097d89 100644
--- a/docs/infrastructure-ovn-setup.md
+++ b/docs/infrastructure-ovn-setup.md
@@ -123,11 +123,22 @@ If there's ever a need to reconfigure a node, simply remove the label and the Da
 
 !!! note
 
-    To upload backups to a Ceph Swift API gateway, edit ovn-backup.config to set
-    `SWIFT_UPLOAD' "true"`, edit the other related options appropriately (i.e.,
-    set the SWIFT_BASE_URL and CONTAINER) and put the username and secret key of
-    the account to use in `swift-account.env` before running `kubectl apply` an
-    indicated above.
+    To upload backups to Swift with tempauth, edit
+    /opt/genestack/kustomize/ovn/ovn-backup/ovn-backup.config to set
+    `SWIFT_TEMPAUTH_UPLOAD' "true"`, edit the other related options
+    appropriately (i.e., set the CONTAINER) and fill the ST_AUTH, ST_USER, and
+    ST_KEY as appropriate for the Swift CLI client in the `swift-tempauth.env`
+    file and then run:
+
+    kubectl apply -k /opt/genestack/kustomize/ovn/ovn-backup \
+    --prune -l app=ovn-backup \
+    --prune-allowlist=core/v1/Secret \
+    --prune-allowlist=core/v1/ConfigMap
+
+    If you need to change variables in the future, you can edit the relevant
+    files and use `kubectl` with these prune options to avoid accumulating
+    old ConfigMaps and Secrets from successive `kubectl apply` operations, but
+    you can omit the pruning options if desired.
 
 ## Centralize `kube-ovn-controller` pods
 
diff --git a/kustomize/ovn/kustomization.yaml b/kustomize/ovn/kustomization.yaml
index 0205b83f..84a68f18 100644
--- a/kustomize/ovn/kustomization.yaml
+++ b/kustomize/ovn/kustomization.yaml
@@ -1,17 +1,2 @@
-secretGenerator:
-  - name: ovn-backup-swift-account
-    namespace: kube-system
-    envs:
-      - swift-account.env
-configMapGenerator:
-  - name: ovn-backup-script
-    namespace: kube-system
-    files:
-    - ovn-backup.sh
-  - name: ovn-backup-config
-    namespace: kube-system
-    envs:
-    - ovn-backup.config
 resources:
   - ovn-setup.yaml
-  - ovn-backup.yaml
diff --git a/kustomize/ovn/ovn-backup.sh b/kustomize/ovn/ovn-backup.sh
deleted file mode 100644
index 2bda515e..00000000
--- a/kustomize/ovn/ovn-backup.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/bin/bash
-
-if [[ "$LOG_LEVEL" == "DEBUG" ]]
-then
-    set -x
-fi
-
-SWIFT_CONTAINER_BASE_URL="$SWIFT_BASE_URL/swift/v1"
-export SWIFT_CONTAINER_BASE_URL
-
-log_level() {
-    local LEVEL="$1"
-    case "$LEVEL" in
-      DEBUG)
-        echo 5
-        ;;
-      INFO)
-        echo 4
-        ;;
-      WARNING)
-        echo 3
-        ;;
-      ERROR)
-        echo 2
-        ;;
-      CRITICAL)
-        echo 1
-        ;;
-      *)
-        exit 3
-        ;;
-    esac
-}
-export -f log_level
-
-log_line() {
-    local LEVEL
-    LEVEL="$(log_level "$1")"
-    if [[ "$LEVEL" -ge "$(log_level "$LOG_LEVEL")" ]]
-    then
-        local line
-        line=$(date +"%b %d %H:%M:%S $*")
-        echo "$line" | tee -a "$LOG_FILE"
-    fi
-}
-export -f log_line # exported for upload_file
-
-# Delete old backup files on volume.
-cd "$BACKUP_DIR" || exit 2
-find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete;
-
-# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR
-YMD="$(date +"%Y/%m/%d")"
-mkdir -p "$YMD" && cd "$YMD" || exit 2 # kubectl-ko creates backups in $PWD, so we cd first.
-/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed"
-/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed"
-
-if [[ "$SWIFT_UPLOAD" != "true" ]]
-then
-    exit 0
-fi
-
-# Everything from here forward deals with uploading to a Ceph Swift API gateway.
-
-cd "$BACKUP_DIR" || exit 2
-CURL="$(which curl)"
-export CONTAINER CURL # these need to reach the subshell below used with `find`
-HEADER_TEMP_FILE=$(mktemp /tmp/headers.XXXXXXXX)
-$CURL -sS -D "$HEADER_TEMP_FILE" -H "X-Auth-User: $USERNAME" -H "X-Auth-Key: $SECRET_KEY" "$SWIFT_BASE_URL/auth/v1.1"
-sed -i -e 's/\r//g' "$HEADER_TEMP_FILE" # strip carriage returns
-token=$(awk '/X-Auth-Token/ { print $2 }' "$HEADER_TEMP_FILE")
-rm "$HEADER_TEMP_FILE"
-export token
-
-# wrap curl with some things we will always use
-curl_wrap() {
-    $CURL -sS -H "X-Auth-Token: $token" "$@"
-}
-export -f curl_wrap
-
-# Create the container if it doesn't exist
-check_container=$(curl_wrap -o /dev/null -w "%{http_code}" "$SWIFT_CONTAINER_BASE_URL/$CONTAINER")
-if ! [[ "$check_container" =~ 20[0-9] ]]
-then
-  curl_wrap -X PUT "$SWIFT_CONTAINER_BASE_URL/$CONTAINER"
-fi
-
-# upload_file uploads $1 to the container
-upload_file() {
-    FILE="$1"
-    local curl_return
-    curl_return=$(curl_wrap -w "%{http_code}" \
-      -X PUT "${SWIFT_CONTAINER_BASE_URL}/${CONTAINER}/$FILE" -T "$FILE")
-    if [[ "$curl_return" == "201" ]]
-    then
-      log_line INFO "SUCCESSFUL UPLOAD $FILE"
-    else
-      log_line ERROR "FAILURE API returned $curl_return uploading $FILE (expected 201)"
-    fi
-}
-export -f upload_file
-
-# find created backups and upload them
-cd "$BACKUP_DIR" || exit 2
-# unusual find syntax to use an exported function from the shell
-find "$YMD" -type f -exec bash -c 'upload_file "$0"' {} \;
diff --git a/kustomize/ovn/ovn-backup/kustomization.yaml b/kustomize/ovn/ovn-backup/kustomization.yaml
new file mode 100644
index 00000000..1603f82b
--- /dev/null
+++ b/kustomize/ovn/ovn-backup/kustomization.yaml
@@ -0,0 +1,16 @@
+commonLabels:
+  app: ovn-backup
+namespace: kube-system
+secretGenerator:
+  - name: ovn-backup-swift-tempauth-account
+    envs:
+      - swift-tempauth.env
+configMapGenerator:
+  - name: ovn-backup-script
+    files:
+    - ovn-backup.sh
+  - name: ovn-backup-config
+    envs:
+    - ovn-backup.config
+resources:
+  - ovn-backup.yaml
diff --git a/kustomize/ovn/ovn-backup.config b/kustomize/ovn/ovn-backup/ovn-backup.config
similarity index 59%
rename from kustomize/ovn/ovn-backup.config
rename to kustomize/ovn/ovn-backup/ovn-backup.config
index 0c37a232..d377c253 100644
--- a/kustomize/ovn/ovn-backup.config
+++ b/kustomize/ovn/ovn-backup/ovn-backup.config
@@ -6,11 +6,7 @@ BACKUP_DIR=/backup
 LOG_FILE=/backup/upload.log
 LOG_LEVEL=INFO
 
-# From here forward, variables for uploading to a Ceph Swift interface.
-SWIFT_UPLOAD=false
-SWIFT_BASE_URL=http://FIX_ME:8081
-
-# Nothing after this line makes any difference unless you used
-# SWIFT_UPLOAD: "true"
-# above.
+# From here forward, variables for uploading to Swift with tempauth
+SWIFT_TEMPAUTH_UPLOAD=false
+# If you change this to "true", set the variables in swift-tempauth.env
 CONTAINER=test-ovn-backup
diff --git a/kustomize/ovn/ovn-backup/ovn-backup.sh b/kustomize/ovn/ovn-backup/ovn-backup.sh
new file mode 100644
index 00000000..a3c25083
--- /dev/null
+++ b/kustomize/ovn/ovn-backup/ovn-backup.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+if [[ "$LOG_LEVEL" == "DEBUG" ]]
+then
+    set -x
+fi
+
+log_level() {
+    local LEVEL="$1"
+    case "$LEVEL" in
+      DEBUG)
+        echo 5
+        ;;
+      INFO)
+        echo 4
+        ;;
+      WARNING)
+        echo 3
+        ;;
+      ERROR)
+        echo 2
+        ;;
+      CRITICAL)
+        echo 1
+        ;;
+      *)
+        exit 3
+        ;;
+    esac
+}
+export -f log_level
+
+log_line() {
+    local LEVEL
+    LEVEL="$(log_level "$1")"
+    if [[ "$LEVEL" -le "$(log_level "$LOG_LEVEL")" ]]
+    then
+        local line
+        line=$(date +"%b %d %H:%M:%S $*")
+        echo "$line" | tee -a "$LOG_FILE"
+    fi
+}
+export -f log_line # exported for upload_file
+
+# Delete old backup files on volume.
+cd "$BACKUP_DIR" || exit 2
+[[ -e "$BACKUP_DIR/last_upload" ]] || touch "$BACKUP_DIR/last_upload" || exit 3
+find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete;
+
+# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR
+YMD="$(date +"%Y/%m/%d")"
+# kubectl-ko creates backups in $PWD, so we cd first.
+mkdir -p "$YMD" && cd "$YMD" || exit 2
+/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed"
+/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed"
+
+if [[ "$SWIFT_TEMPAUTH_UPLOAD" != "true" ]]
+then
+    exit 0
+fi
+
+# Everything from here forward deals with uploading to a Swift with tempauth.
+
+cd "$BACKUP_DIR" || exit 2
+
+# Make a working "swift" command
+SWIFT="kubectl -n openstack exec -i openstack-admin-client --
+env -i ST_AUTH=$ST_AUTH ST_USER=$ST_USER ST_KEY=$ST_KEY
+/var/lib/openstack/bin/swift"
+export SWIFT
+
+# Create the container if it doesn't exist
+if ! $SWIFT stat "$CONTAINER" > /dev/null
+then
+  $SWIFT post "$CONTAINER"
+fi
+
+# upload_file uploads $1 to the container
+upload_file() {
+    FILE="$1"
+    # Using OBJECT_NAME instead of FILE every time doesn't change the behavior,
+    # but stops shellcheck from identifying this as trying to read and write
+    # the same file.
+    OBJECT_NAME="$FILE"
+    if $SWIFT upload "$CONTAINER" --object-name "$OBJECT_NAME" - < "$FILE"
+    then
+      log_line INFO "SUCCESSFUL UPLOAD $FILE as object $OBJECT_NAME"
+    else
+      log_line ERROR "FAILURE API swift exited $? uploading $FILE as $OBJECT_NAME"
+    fi
+}
+export -f upload_file
+
+# find created backups and upload them
+cd "$BACKUP_DIR" || exit 2
+# unusual find syntax to use an exported function from the shell
+find "$YMD" -type f -newer "$BACKUP_DIR/last_upload" \
+-exec bash -c 'upload_file "$0"' {} \;
+touch "$BACKUP_DIR/last_upload"
diff --git a/kustomize/ovn/ovn-backup.yaml b/kustomize/ovn/ovn-backup/ovn-backup.yaml
similarity index 94%
rename from kustomize/ovn/ovn-backup.yaml
rename to kustomize/ovn/ovn-backup/ovn-backup.yaml
index 3fc1e6f9..edeafe63 100644
--- a/kustomize/ovn/ovn-backup.yaml
+++ b/kustomize/ovn/ovn-backup/ovn-backup.yaml
@@ -9,7 +9,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  namespace: kube-system
   name: ovndb-backup
 spec:
   accessModes:
@@ -23,7 +22,6 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: ovn-snapshot-cron
-  namespace: kube-system
 spec:
   schedule: "0 0 * * *"
   concurrencyPolicy: Forbid
@@ -50,7 +48,7 @@ spec:
               - configMapRef:
                   name: ovn-backup-config
               - secretRef:
-                  name: ovn-backup-swift-account
+                  name: ovn-backup-swift-tempauth-account
             command: ["/backup-script/ovn-backup.sh"]
             image: docker.io/kubeovn/kube-ovn:v1.11.5
             imagePullPolicy: IfNotPresent
diff --git a/kustomize/ovn/ovn-backup/swift-tempauth.env b/kustomize/ovn/ovn-backup/swift-tempauth.env
new file mode 100644
index 00000000..39f9231d
--- /dev/null
+++ b/kustomize/ovn/ovn-backup/swift-tempauth.env
@@ -0,0 +1,6 @@
+# Set variables as appropriate for the Swift client for tempauth.
+# ST_AUTH for the auth URL,
+# e.g., https://tempauth.environment.yourdomain.invalid/auth/v1.0
+ST_AUTH=url
+ST_USER=username
+ST_KEY=passwordOrKey
diff --git a/kustomize/ovn/swift-account.env b/kustomize/ovn/swift-account.env
deleted file mode 100644
index d46f0326..00000000
--- a/kustomize/ovn/swift-account.env
+++ /dev/null
@@ -1,2 +0,0 @@
-USERNAME=username
-SECRET_KEY=secret_key