diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index e47d8d88c..c8dfef417 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -12,6 +12,17 @@ jobs:
       contents: write
       pull-requests: write
     name: Backport
+    # Only react to merged PRs for security reasons.
+    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
     steps:
       - name: GitHub App token
         id: github_app_token
@@ -19,10 +30,14 @@ jobs:
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          # opensearch-trigger-bot installation ID
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v1.1.4
+        uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          files_to_skip: "CHANGELOG.md"
+          labels_template: "<%= JSON.stringify([...labels, 'autocut']) %>"
+          failure_labels: "failed backport"