Merge pull request #54 from galaxyproject/prune-snapshots

Add a script for pruning snapshots on release managers and optionally schedule with cron

Author: natefoo

README.md

@@ -89,6 +89,23 @@ cvmfs_repositories:
       - KEY=val
 ```
 
+For Stratum 0 / Release Managers, you can automatically prune older snapshots using the `prune_snapshots_time`, a hash
+having keys that correspond to the [cron module
+options](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/cron_module.html). If
+`prune_snapshots_time` is unset, then snapshots are not automatically pruned.
+
+```
+cvmfs_repositories:
+  - repository: repo.example.org
+    owner: user1
+    prune_snapshots_count: 20
+    prune_snapshots_time:
+       special_time: daily
+```
+
+The per-repository `prune_snapshots_count` option defaults to the value of `cvmfs_stratum0_prune_snapshots_count` in
+[defaults/main.yml][defaults] if unset.
+
 ## Server variables
 
 variable | type | description

defaults/main.yml

@@ -41,6 +41,9 @@ cvmfs_stratum1_cache_mem: 128 # MB
 cvmfs_stratum1_snapshot_time:
   special_time: hourly
 
+# Number of snapshots to keep. Per the documentation, the recommended count is no more than 50.
+cvmfs_stratum0_prune_snapshots_count: 50
+
 # Whether the client or server should be upgraded or just installed if missing
 cvmfs_upgrade_client: false
 cvmfs_upgrade_server: false

files/cvmfs_prune_snapshots.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# Prune named snapshots on CVMFS release managers by age and count
+#
+set -euo pipefail
+
+COUNT=50
+VERBOSE=false
+DRYRUN=false
+MUTEX="${HOME}/.updaterepo.lock"
+MUTEX_ACQUIRED=false
+
+function help() {
+    cat <<EOF
+usage: $0 [options] REPOSITORY
+options:
+  -h        this help message
+  -v        verbose ouptut
+  -c COUNT  number of snapshots to keep (default: ${COUNT})
+  -n        no changes (dry run, only show what would be done)
+EOF
+}
+
+function trap_handler() {
+    $MUTEX_ACQUIRED && { rmdir "$MUTEX"; }
+    return 0
+}
+trap "trap_handler" SIGTERM SIGINT ERR EXIT
+
+while getopts ":c:hnv" opt; do
+    case "$opt" in
+        c)
+            COUNT="$OPTARG"
+            ;;
+        h)
+            help
+            exit 0
+            ;;
+        n)
+            DRYRUN=true
+            ;;
+        v)
+            VERBOSE=true
+            ;;
+        *)
+            echo "ERROR: Unknown option: ${opt}. See '-h' for help."
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${1:-}" ]; then
+    echo "ERROR: repository argument is required. See '-h' for help."
+    exit 1
+fi
+
+REPO="$1"
+
+echo "Keeping newest ${COUNT} named snapshots in repository: ${REPO}"
+
+if ! $DRYRUN; then
+    mkdir "$MUTEX"
+    MUTEX_ACQUIRED=true
+fi
+
+tag_args=($(cvmfs_server tag -lx "$REPO" | sort -nk 5 | head -n -"$COUNT" | awk '{print $1}' | sed 's/^/-r /'))
+
+if [ "${#tag_args[@]}" -eq 0 ]; then
+    echo "no tags to remove"
+    exit 0
+fi
+
+if $DRYRUN; then
+    echo cvmfs_server tag -f "${tag_args[@]}" "$REPO"
+else
+    cvmfs_server tag -f "${tag_args[@]}" "$REPO"
+fi

tasks/stratum0.yml

@@ -90,3 +90,27 @@
 - name: Include garbage collection tasks
   ansible.builtin.include_tasks: gc.yml
   when: cvmfs_gc_enabled
+
+- name: Install cvmfs_prune_snapshots
+  copy:
+    src: cvmfs_prune_snapshots.sh
+    dest: /usr/local/bin/cvmfs_prune_snapshots
+    mode: 0755
+
+- name: Schedule snapshot pruning
+  ansible.builtin.cron:
+    name: cvmfs_prune_snapshots_{{ item.repository }}
+    cron_file: ansible_cvmfs_stratum0_prune_snapshots
+    user: "{{ item.owner | default('root') }}"
+    job: >-
+      output=$(/usr/local/bin/cvmfs_prune_snapshots
+      -c {{ item.prune_snapshots_count | default(cvmfs_stratum0_prune_snapshots_count) }}
+      '{{ item.repository }}' 2>&1) || echo "$output"
+    hour: "{{ item.prune_snapshots_time.hour | default(omit) }}"
+    minute: "{{ item.prune_snapshots_time.minute | default(omit) }}"
+    day: "{{ item.prune_snapshots_time.day | default(omit) }}"
+    month: "{{ item.prune_snapshots_time.month | default(omit) }}"
+    weekday: "{{ item.prune_snapshots_time.weekday | default(omit) }}"
+    special_time: "{{ item.prune_snapshots_time.special_time | default(omit) }}"
+  loop: "{{ cvmfs_repositories }}"
+  when: item.prune_snapshots_time is defined