Switch to Swift tempauth for OVN backups (#304)

* Switch OVN backups to using Swift.

JIRA:OSPC-432

* Move OVN backup stuff to a separate directory.

This helps make it easy to use:

commonLabels:
  app: ovn-backup

to label all of the resources in kustomization.yaml, and the backup
functionality started cluttering up the main directory.

JIRA:OSPC-432

* For OVN backups, add script logic to upload new files since last upload.

We have MariaDB backups going every 6 hours now, and previous logic only
allowed for once-a-day backups.

JIRA:OSPC-432

* For OVN backups, make kustomize put the namespace for all resources.

JIRA:OSPC-432

* For OVN backups, switch Swift upload to Swift tempauth upload.

JIRA:OSPC-432

docs/infrastructure-ovn-setup.md

@@ -123,11 +123,22 @@ If there's ever a need to reconfigure a node, simply remove the label and the Da
 
 !!! note
 
-    To upload backups to a Ceph Swift API gateway, edit ovn-backup.config to set
-    `SWIFT_UPLOAD' "true"`, edit the other related options appropriately (i.e.,
-    set the SWIFT_BASE_URL and CONTAINER) and put the username and secret key of
-    the account to use in `swift-account.env` before running `kubectl apply` an
-    indicated above.
+    To upload backups to Swift with tempauth, edit
+    /opt/genestack/kustomize/ovn/ovn-backup/ovn-backup.config to set
+    `SWIFT_TEMPAUTH_UPLOAD' "true"`, edit the other related options
+    appropriately (i.e., set the CONTAINER) and fill the ST_AUTH, ST_USER, and
+    ST_KEY as appropriate for the Swift CLI client in the `swift-tempauth.env`
+    file and then run:
+
+    kubectl apply -k /opt/genestack/kustomize/ovn/ovn-backup \
+    --prune -l app=ovn-backup \
+    --prune-allowlist=core/v1/Secret \
+    --prune-allowlist=core/v1/ConfigMap
+
+    If you need to change variables in the future, you can edit the relevant
+    files and use `kubectl` with these prune options to avoid accumulating
+    old ConfigMaps and Secrets from successive `kubectl apply` operations, but
+    you can omit the pruning options if desired.
 
 ## Centralize `kube-ovn-controller` pods
 

kustomize/ovn/kustomization.yaml

@@ -1,17 +1,2 @@
-secretGenerator:
-  - name: ovn-backup-swift-account
-    namespace: kube-system
-    envs:
-      - swift-account.env
-configMapGenerator:
-  - name: ovn-backup-script
-    namespace: kube-system
-    files:
-    - ovn-backup.sh
-  - name: ovn-backup-config
-    namespace: kube-system
-    envs:
-    - ovn-backup.config
 resources:
   - ovn-setup.yaml
-  - ovn-backup.yaml

kustomize/ovn/ovn-backup/kustomization.yaml

@@ -0,0 +1,16 @@
+commonLabels:
+  app: ovn-backup
+namespace: kube-system
+secretGenerator:
+  - name: ovn-backup-swift-tempauth-account
+    envs:
+      - swift-tempauth.env
+configMapGenerator:
+  - name: ovn-backup-script
+    files:
+    - ovn-backup.sh
+  - name: ovn-backup-config
+    envs:
+    - ovn-backup.config
+resources:
+  - ovn-backup.yaml

kustomize/ovn/ovn-backup.config → kustomize/ovn/ovn-backup/ovn-backup.config

@@ -6,11 +6,7 @@ BACKUP_DIR=/backup
 LOG_FILE=/backup/upload.log
 LOG_LEVEL=INFO
 
-# From here forward, variables for uploading to a Ceph Swift interface.
-SWIFT_UPLOAD=false
-SWIFT_BASE_URL=http://FIX_ME:8081
-
-# Nothing after this line makes any difference unless you used
-# SWIFT_UPLOAD: "true"
-# above.
+# From here forward, variables for uploading to Swift with tempauth
+SWIFT_TEMPAUTH_UPLOAD=false
+# If you change this to "true", set the variables in swift-tempauth.env
 CONTAINER=test-ovn-backup

kustomize/ovn/ovn-backup/ovn-backup.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+if [[ "$LOG_LEVEL" == "DEBUG" ]]
+then
+    set -x
+fi
+
+log_level() {
+    local LEVEL="$1"
+    case "$LEVEL" in
+      DEBUG)
+        echo 5
+        ;;
+      INFO)
+        echo 4
+        ;;
+      WARNING)
+        echo 3
+        ;;
+      ERROR)
+        echo 2
+        ;;
+      CRITICAL)
+        echo 1
+        ;;
+      *)
+        exit 3
+        ;;
+    esac
+}
+export -f log_level
+
+log_line() {
+    local LEVEL
+    LEVEL="$(log_level "$1")"
+    if [[ "$LEVEL" -le "$(log_level "$LOG_LEVEL")" ]]
+    then
+        local line
+        line=$(date +"%b %d %H:%M:%S $*")
+        echo "$line" | tee -a "$LOG_FILE"
+    fi
+}
+export -f log_line # exported for upload_file
+
+# Delete old backup files on volume.
+cd "$BACKUP_DIR" || exit 2
+[[ -e "$BACKUP_DIR/last_upload" ]] || touch "$BACKUP_DIR/last_upload" || exit 3
+find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete;
+
+# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR
+YMD="$(date +"%Y/%m/%d")"
+# kubectl-ko creates backups in $PWD, so we cd first.
+mkdir -p "$YMD" && cd "$YMD" || exit 2
+/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed"
+/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed"
+
+if [[ "$SWIFT_TEMPAUTH_UPLOAD" != "true" ]]
+then
+    exit 0
+fi
+
+# Everything from here forward deals with uploading to a Swift with tempauth.
+
+cd "$BACKUP_DIR" || exit 2
+
+# Make a working "swift" command
+SWIFT="kubectl -n openstack exec -i openstack-admin-client --
+env -i ST_AUTH=$ST_AUTH ST_USER=$ST_USER ST_KEY=$ST_KEY
+/var/lib/openstack/bin/swift"
+export SWIFT
+
+# Create the container if it doesn't exist
+if ! $SWIFT stat "$CONTAINER" > /dev/null
+then
+  $SWIFT post "$CONTAINER"
+fi
+
+# upload_file uploads $1 to the container
+upload_file() {
+    FILE="$1"
+    # Using OBJECT_NAME instead of FILE every time doesn't change the behavior,
+    # but stops shellcheck from identifying this as trying to read and write
+    # the same file.
+    OBJECT_NAME="$FILE"
+    if $SWIFT upload "$CONTAINER" --object-name "$OBJECT_NAME" - < "$FILE"
+    then
+      log_line INFO "SUCCESSFUL UPLOAD $FILE as object $OBJECT_NAME"
+    else
+      log_line ERROR "FAILURE API swift exited $? uploading $FILE as $OBJECT_NAME"
+    fi
+}
+export -f upload_file
+
+# find created backups and upload them
+cd "$BACKUP_DIR" || exit 2
+# unusual find syntax to use an exported function from the shell
+find "$YMD" -type f -newer "$BACKUP_DIR/last_upload" \
+-exec bash -c 'upload_file "$0"' {} \;
+touch "$BACKUP_DIR/last_upload"

kustomize/ovn/ovn-backup.yaml → kustomize/ovn/ovn-backup/ovn-backup.yaml

@@ -9,7 +9,6 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  namespace: kube-system
   name: ovndb-backup
 spec:
   accessModes:
@@ -23,7 +22,6 @@ apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: ovn-snapshot-cron
-  namespace: kube-system
 spec:
   schedule: "0 0 * * *"
   concurrencyPolicy: Forbid
@@ -50,7 +48,7 @@ spec:
               - configMapRef:
                   name: ovn-backup-config
               - secretRef:
-                  name: ovn-backup-swift-account
+                  name: ovn-backup-swift-tempauth-account
             command: ["/backup-script/ovn-backup.sh"]
             image: docker.io/kubeovn/kube-ovn:v1.11.5
             imagePullPolicy: IfNotPresent

kustomize/ovn/ovn-backup/swift-tempauth.env

@@ -0,0 +1,6 @@
+# Set variables as appropriate for the Swift client for tempauth.
+# ST_AUTH for the auth URL,
+# e.g., https://tempauth.environment.yourdomain.invalid/auth/v1.0
+ST_AUTH=url
+ST_USER=username
+ST_KEY=passwordOrKey