Merge pull request #1 from alalkamys/feat/vault-kv-migrate-role

Release: vault_kv_migrate role

.ansible-lint

@@ -0,0 +1,9 @@
+---
+exclude_paths:
+  - .github/
+  - tests
+  - .ansible-lint
+
+skip_list:
+  - yaml[line-length]
+  - name[template]

.github/workflows/ci.yaml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - "**"
+    paths-ignore:
+      - "docs/**"
+      - "**/*.md"
+      - ".gitignore"
+      - "LICENSE"
+  pull_request:
+    branches:
+      - main
+      - "releases/**"
+
+defaults:
+  run:
+    working-directory: "alalkamys.vault_kv_migrate"
+
+# ensure that only a single job or workflow using the same concurrency group will run at a time
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # cancel previously running builds in a PR on new pushes
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+# set default permissions granted to the GITHUB_TOKEN to read only to follow least privilege principle
+permissions: read-all
+
+jobs:
+  lint:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v2
+        with:
+          path: "alalkamys.vault_kv_migrate"
+
+      - name: Set up Python 3.
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install ansible-core and ansible-lint.
+        run: pip3 install ansible-core ansible-lint
+
+      - name: ansible-lint.
+        run: ansible-lint -c .ansible-lint

.github/workflows/release.yaml

@@ -0,0 +1,44 @@
+name: Release
+
+"on":
+  push:
+    tags:
+      - "v*"
+      - "v*.*"
+      - "v*.*.*"
+
+defaults:
+  run:
+    working-directory: "alalkamys.vault_kv_migrate"
+
+# ensure that only a single job or workflow using the same concurrency group will run at a time
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+# set default permissions granted to the GITHUB_TOKEN to read only to follow least privilege principle
+permissions: read-all
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    # set timeout to 15 mins max to decrease hanging jobs issues, default is 6 Hrs
+    timeout-minutes: 15
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v2
+        with:
+          path: "alalkamys.vault_kv_migrate"
+
+      - name: Set up Python 3.
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install Ansible.
+        run: pip3 install ansible-core
+
+      - name: Trigger a new import on Galaxy.
+        run: >-
+          ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }}
+          $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2)

.gitignore

@@ -158,3 +158,15 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# ignore .DS_Store
+.DS_Store
+
+# ignore vscode temp generated files
+.vscode/*
+
+# ignore tests/vars
+tests/vars
+
+# ignore secrets.json
+secrets.json

README.md

@@ -0,0 +1,138 @@
+# Vault KV Migrate Role
+
+[![CI](https://github.com/alalkamys/ansible-role-vault-kv-migrate/actions/workflows/ci.yaml/badge.svg)](https://github.com/alalkamys/ansible-role-vault-kv-migrate/actions/workflows/ci.yaml)
+[![License](https://img.shields.io/badge/license-MIT%20License-brightgreen.svg)](https://opensource.org/licenses/MIT)
+[![Ansible Role](https://img.shields.io/badge/ansible%20role-alalkamys.vault_kv_migrate-blue.svg)](https://galaxy.ansible.com/alalkamys/vault_kv_migrate/)
+[![GitHub tag](https://img.shields.io/github/tag/alakamys/ansible-role-vault-kv-migrate.svg)](https://github.com/alalkamys/ansible-role-vault-kv-migrate/tags)
+
+## Overview
+
+The `vault_kv_migrate` role automates the migration of secrets from one HashiCorp Vault Key-Value (KV) engine to multiple engines. It can also export HashiCorp KV secrets for a given path recursively and save them to a file named `'secrets.json'` for backups. `vault_kv_migrate` is perfect for operational tasks where you need to either replicate HashiCorp KV secrets to one or more Vault servers, KV engines within the same Vault server or a mix of both. It is also handful when you want to export the KV secrets to your machine as a backup.
+
+`vault_kv_migrate` can also write migrate KV secrets of HashiCorp Vault sitting behind [Cloudflare Zero Trust](https://developers.cloudflare.com/cloudflare-one/).
+
+> **Note:** `vault_kv_migrate` is meant for operation tasks.
+
+## Requirements
+
+- Ansible 2.11.5 or higher
+- jmespath 0.10.0 or higher
+- vault CLI v1.15 or higher
+- HashiCorp Vault installed and configured
+
+## Role Variables
+
+| Variable                                    | Default Value                             | Description                                                                                                                                                          |
+| ------------------------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `vault_kv_migrate_vault_api_version`        | `v1`                                      | Vault API version to use.                                                                                                                                            |
+| `vault_kv_migrate_vault_api_validate_certs` | `no`                                      | Whether to validate SSL certificates for Vault API requests. Set to `yes` to enable certificate validation.                                                          |
+| `vault_kv_migrate_remove_backup`            | `no`                                      | Whether to remove `'secrets.json'` backup file after migration. Set to `yes` to remove the local backup file after the migration.                                    |
+| `vault_kv_migrate_cf_token`                 | `""`                                      | Cloudflare token for Zero trust authentication. If not used, keep it empty.                                                                                          |
+| `vault_kv_migrate_src_vault_addr`           | `"http://localhost:8200"`                 | Address of the source Vault server.                                                                                                                                  |
+| `vault_kv_migrate_src_vault_token`          | `""`                                      | Token for authentication with the source Vault server.                                                                                                               |
+| `vault_kv_migrate_src_vault_namespace`      | `""`                                      | Namespace for the source Vault server.                                                                                                                               |
+| `vault_kv_migrate_src_engine`               | `secret`                                  | Source Vault KV engine from which secrets will be migrated. Don't add trailing `/` to the engine.                                                                    |
+| `vault_kv_migrate_src_secret_path`          | `""`                                      | Path to the source secret within the source engine. if the value is `""` `vault_kv_migrate` will export/migrate all the secrets under `vault_kv_migrate_src_engine`. |
+| `vault_kv_migrate_dest_kv_engines`          | [See example playbook](#example-playbook) | List of destination Vault KV engines with configurations. See example playbook for structure.                                                                        |
+
+## Example Playbook
+
+```yaml
+- hosts: localhost
+  become: no
+  roles:
+    - vault_kv_migrate
+  vars:
+    vault_kv_migrate_vault_api_version: "v1"
+    vault_kv_migrate_vault_api_validate_certs: no
+    vault_kv_migrate_remove_backup: no
+    vault_kv_migrate_cf_token: ""
+    vault_kv_migrate_src_vault_addr: "http://localhost:8200"
+    vault_kv_migrate_src_vault_token: ""
+    vault_kv_migrate_src_vault_namespace: ""
+    vault_kv_migrate_src_engine: "secret"
+    vault_kv_migrate_src_secret_path: ""
+    vault_kv_migrate_dest_kv_engines:
+      - vault_addr: "http://localhost:8200"
+        vault_token: ""
+        vault_namespace: ""
+        engine: "secret2"
+```
+
+## Role Workflow
+
+### 1. Export Source Secrets
+
+The role first exports secrets from the specified source Vault KV engine and path. It securely retrieves the secrets and saves them to a temporary file named `'secrets.json'`.
+
+### 2. Transfer to Destination Engines
+
+The exported secrets are then transferred to multiple destination Vault KV engines. For each secret, the role makes secure POST requests to the corresponding destination paths in the destination Vault KV engines. This ensures that the secrets are securely and accurately transferred.
+
+### 3. Backup and Cleanup
+
+After the migration is completed, the backup file `'secrets.json'` can be removed based on the value of the `vault_kv_migrate_remove_backup` variable. Removing the backup file is optional and can be configured as per your requirements.
+
+## Role Tags
+
+| Tag             | Action                                                                                                       | Example                                        |
+| --------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------- |
+| `export`        | exports the kv secrets only and saves it to your machine in `secrets.json`                                   | ansible-playbook site.yml --tags export        |
+| `write_secrets` | writes secrets in `secrets.json` found under your `{{ playbook_dir }}` to your list of Vault KV engines only | ansible-playbook site.yml --tags write_secrets |
+| `remove_backup` | removes `{{ playbook_dir }}/secrets.json` only                                                               | ansible-playbook site.yml --tags remove_backup |
+
+## Secrets Backup File
+
+The exported secrets data will be stored in `{{ playboook_dir }}/secrets.json`, an example can be seen down below
+
+```json
+[
+  {
+    "path": "secret/path/to/secret1",
+    "value": {
+      "data": {
+        "data": {
+          "key": "value",
+        },
+        "metadata": {
+          "created_time": "2023-11-04T14:30:44.5094809Z",
+          "custom_metadata": null,
+          "deletion_time": "",
+          "destroyed": false,
+          "version": 1
+        }
+      }
+    }
+  },
+  {
+    "path": "secret/path/to/secret2",
+    "value": {
+      "data": {
+        "data": {
+          "key1": "value",
+          "key2": "value",
+        },
+        "metadata": {
+          "created_time": "2023-11-04T14:30:45.217227213Z",
+          "custom_metadata": null,
+          "deletion_time": "",
+          "destroyed": false,
+          "version": 1
+        }
+      }
+    }
+  }
+]
+```
+
+## License
+
+This role is licensed under the MIT License. For more details, refer to the [LICENSE](LICENSE) file.
+
+## Author Information
+
+- **Author:** Shehab El-Deen Alalkamy
+- **Email:** [shehabeldeenalalkamy@gmail.com](mailto:shehabeldeenalalkamy@gmail.com)
+- **GitHub:** [alkamys](https://github.com/alkamys)
+
+For more information and updates, please visit the [GitHub repository](https://github.com/alkamys/ansible-role-kv-migrate).

defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Vault API
+vault_kv_migrate_vault_api_version: v1
+vault_kv_migrate_vault_api_validate_certs: false
+
+# Removes 'secrets.json' after migration
+vault_kv_migrate_remove_backup: false
+
+# Cloudflare token
+vault_kv_migrate_cf_token: ""
+
+# Source Vault Server
+vault_kv_migrate_src_vault_addr: "http://localhost:8200"
+vault_kv_migrate_src_vault_token: ""
+vault_kv_migrate_src_vault_namespace: ""
+
+# don't add trailing '/'
+vault_kv_migrate_src_engine: secret
+vault_kv_migrate_src_secret_path: ""
+
+# Destination KV Engines
+vault_kv_migrate_dest_kv_engines:
+  - vault_addr: "http://localhost:8200"
+    vault_token: ""
+    vault_namespace: ""
+    # don't add trailing '/'
+    engine: secret2

files/vault-kv-export.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+readonly ARB_TEMP_SECRETS_FILE="arbitrary_temp_secrets.json"
+readonly TEMP_SECRETS_FILE="temp_secrets.json"
+readonly SECRETS_FILE="secrets.json"
+
+log() {
+    local log_type="$1"
+    local message="$2"
+    local timestamp
+    timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+    echo "[${log_type}] [${timestamp}] ${message}"
+}
+
+log_info() {
+    log "INFO" "$1"
+}
+
+log_error() {
+    log "ERROR" "$1"
+    exit 1
+}
+
+traverse() {
+    local path="$1"
+    local result
+
+    local headers=()
+    if [[ -n "${CF_TOKEN}" || "${CF_TOKEN}" != "" ]]; then
+        headers+=("-header" "cf-access-token=${CF_TOKEN}")
+    fi
+
+    result=$(vault kv list -format=json "${headers[@]}" "${path}" 2>&1) || log_error "Failed to list secrets: ${result}"
+
+    while IFS= read -r secret; do
+        if [[ "${secret}" == */ ]]; then
+            traverse "${path}${secret}"
+        else
+            local secret_data
+            secret_data=$(vault kv get -format=json "${headers[@]}" "${path}${secret}" | jq -r '.data') || log_error "Failed to get secret data: ${secret_data}"
+
+            if [[ "${secret_data}" != "null" ]]; then
+                echo "{\"path\":\"${path}${secret}\",\"value\":{\"data\":${secret_data}}}," >>"${ARB_TEMP_SECRETS_FILE}"
+            fi
+        fi
+    done < <(echo "${result}" | jq -r '.[]')
+}
+
+main() {
+    log_info "Starting secrets retrieval process."
+
+    [[ -f "${ARB_TEMP_SECRETS_FILE}" ]] && rm -f "${ARB_TEMP_SECRETS_FILE}"
+    [[ -f "${TEMP_SECRETS_FILE}" ]] && rm -f "${TEMP_SECRETS_FILE}"
+    [[ -f "${SECRETS_FILE}" ]] && rm -f "${SECRETS_FILE}"
+
+    if [[ -n "${CF_TOKEN}" || "${CF_TOKEN}" != "" ]]; then
+        log_info "CF_TOKEN detected."
+    fi
+
+    local vaults
+    if [[ "$1" ]]; then
+        vaults=("${1%"/"}/")
+        log_info "Retrieving all secrets under ${vaults[*]}.."
+    else
+        local headers=()
+        if [[ -n "${CF_TOKEN}" || "${CF_TOKEN}" != "" ]]; then
+            headers+=("-header" "cf-access-token=${CF_TOKEN}")
+        fi
+        log_info "No secret engine provided. Retrieving all secrets.."
+        result=$(vault secrets list -format=json "${headers[@]}" 2>&1) || log_error "Failed to list secrets engines: ${result}"
+        mapfile -t vaults < <(echo "${result}" | jq -r 'to_entries[] | select(.value.type=="kv") | .key')
+    fi
+
+    for vault in "${vaults[@]}"; do
+        traverse "${vault}"
+    done
+
+    echo "[" >"${TEMP_SECRETS_FILE}"
+    sed '$s/,$//' "${ARB_TEMP_SECRETS_FILE}" >>"${TEMP_SECRETS_FILE}"
+    echo "]" >>"${TEMP_SECRETS_FILE}"
+
+    jq . "${TEMP_SECRETS_FILE}" >"${SECRETS_FILE}"
+    rm "${ARB_TEMP_SECRETS_FILE}" "${TEMP_SECRETS_FILE}"
+
+    log_info "Secrets retrieval completed and saved to ${SECRETS_FILE}"
+}
+
+[[ "$0" == "${BASH_SOURCE[0]}" ]] && main "$@"