feat: SOPS secrets support

deploy_config_generator/__main__.py

@@ -16,6 +16,7 @@
 from deploy_config_generator.template import Template
 from deploy_config_generator.errors import DeployConfigError, DeployConfigGenerationError, ConfigError, VarsReplacementError
 from deploy_config_generator.utils import yaml_dump, show_traceback
+from deploy_config_generator.secrets import Secrets, SecretsException
 
 DISPLAY = None
 SITE_CONFIG = None
@@ -78,7 +79,7 @@ def load_vars_files(varset, vars_dir, patterns, allow_var_references=True):
         varset.read_vars_file(vars_file, allow_var_references=allow_var_references)
 
 
-def load_output_plugins(varset, output_dir, config_version):
+def load_output_plugins(varset, secrets, output_dir, config_version):
     '''
     Find, import, and instantiate all output plugins
     '''
@@ -111,7 +112,7 @@ def load_output_plugins(varset, output_dir, config_version):
                     raise Exception('name specified in OutputPlugin class (%s) does not match file name (%s)' % (cls.NAME, name))
                 DISPLAY.v('Loading plugin %s' % name)
                 # Instantiate plugin class
-                plugins.append(cls(varset, output_dir, config_version))
+                plugins.append(cls(varset, secrets, output_dir, config_version))
             except ConfigError as e:
                 DISPLAY.display('Plugin configuration error: %s: %s' % (name, str(e)))
                 sys.exit(1)
@@ -263,8 +264,26 @@ def main():
     DISPLAY.vvvv()
     DISPLAY.vvvv(yaml_dump(dict(varset), indent=2))
 
+    secrets = Secrets()
+
+    try:
+        for f in SITE_CONFIG.secrets_files:
+            if os.path.exists(f):
+                DISPLAY.v('Loading secrets from %s' % f)
+                secrets.load_secrets_file(f)
+    except SecretsException as e:
+        DISPLAY.display('Error loading secrets file %s' % (e.path,))
+        DISPLAY.v('Stderr:')
+        DISPLAY.v()
+        DISPLAY.v(e.stderr)
+        sys.exit(1)
+
+    DISPLAY.vvvv('Secrets:')
+    DISPLAY.vvvv()
+    DISPLAY.vvvv(yaml_dump(dict(secrets), indent=2))
+
     try:
-        deploy_config = DeployConfig(os.path.join(deploy_dir, SITE_CONFIG.deploy_config_file), varset)
+        deploy_config = DeployConfig(os.path.join(deploy_dir, SITE_CONFIG.deploy_config_file), varset, secrets)
         deploy_config.set_config(varset.replace_vars(deploy_config.get_config()))
     except DeployConfigError as e:
         DISPLAY.display('Error loading deploy config: %s' % str(e))
@@ -280,7 +299,7 @@ def main():
     DISPLAY.vvvv(yaml_dump(deploy_config.get_config(), indent=2))
 
     deploy_config_version = deploy_config.get_version() or SITE_CONFIG.default_config_version
-    output_plugins = load_output_plugins(varset, args.output_dir, deploy_config_version)
+    output_plugins = load_output_plugins(varset, secrets, args.output_dir, deploy_config_version)
 
     DISPLAY.vvv('Available output plugins:')
     DISPLAY.vvv()

deploy_config_generator/deploy_config.py

@@ -12,8 +12,9 @@ class DeployConfig(object):
     _path = None
     _version = None
 
-    def __init__(self, path, varset):
+    def __init__(self, path, varset, secrets):
         self._vars = varset
+        self._secrets = secrets
         self._display = Display()
         self.load(path)
 
@@ -68,7 +69,7 @@ def apply_default_apps(self, default_apps):
                     condition = app[CONDITION_KEY]
                     del app[CONDITION_KEY]
                     template = Template()
-                    tmp_vars = dict(VARS=dict(self._vars))
+                    tmp_vars = dict(VARS=dict(self._vars), SECRETS=dict(self._secrets))
                     # Continue to next item if we have a condition and it evaluated to False
                     if not template.evaluate_condition(condition, tmp_vars):
                         continue

deploy_config_generator/output/__init__.py

@@ -30,8 +30,9 @@ class OutputPluginBase(object):
     )
     PRIORITY = 1
 
-    def __init__(self, varset, output_dir, config_version):
+    def __init__(self, varset, secrets, output_dir, config_version):
         self._vars = varset
+        self._secrets = secrets
         self._output_dir = output_dir
         self._display = Display()
         self._template = Template()
@@ -201,6 +202,8 @@ def build_app_vars(self, index, app, path=''):
             'APP': self.merge_with_field_defaults(app),
             # Parsed vars
             'VARS': dict(self._vars),
+            # Secrets
+            'SECRETS': dict(self._secrets),
         }
         return app_vars
 

deploy_config_generator/secrets.py

@@ -0,0 +1,28 @@
+import subprocess
+
+from deploy_config_generator.utils import yaml_load
+
+SOPS_BINARY = 'sops'
+
+
+class SecretsException(Exception):
+
+    def __init__(self, path, stderr):
+        self.path = path
+        self.stderr = stderr
+        super().__init__('Failed to decrypt secrets')
+
+
+class Secrets(dict):
+
+    '''
+    This class manages a set of SOPS-encrypted secrets
+    '''
+
+    def load_secrets_file(self, path):
+        cp = subprocess.run([SOPS_BINARY, 'decrypt', '--output-type=yaml', path], capture_output=True)
+        if cp.returncode != 0:
+            raise SecretsException(path, cp.stderr)
+        # Parse decrypted YAML from SOPS
+        data = yaml_load(cp.stdout)
+        self.update(data)

deploy_config_generator/site_config.py

@@ -49,6 +49,8 @@ class SiteConfig(object, metaclass=Singleton):
         'default_apps': {},
         # Default vars
         'default_vars': {},
+        # SOPS secrets files to load
+        'secrets_files': [],
     }
 
     def __init__(self, env=None):
@@ -114,6 +116,11 @@ def load_file(self, path):
                 include_data = self.load_file(include_path)
                 data = dict_merge(data, include_data)
             del data['include']
+        if 'secrets_files' in data:
+            for idx, secrets_file in enumerate(data['secrets_files']):
+                if not secrets_file.startswith('/'):
+                    # Normalize secrets file path based on location of parent file
+                    data['secrets_files'][idx] = os.path.join(os.path.dirname(path), secrets_file)
         return data
 
     def load(self, path):

scripts/ci.sh

@@ -1,5 +1,12 @@
 #!/bin/bash -x
 
+# Install age
+sudo apt install -y age
+
+# Install sops
+sudo curl -Lo /usr/local/bin/sops https://github.com/getsops/sops/releases/download/v3.11.0/sops-v3.11.0.linux.$(uname -m | sed -e 's:x86_64:amd64:' -e 's:aarch64:arm64:')
+sudo chmod a+x /usr/local/bin/sops
+
 # Install/run tox
 pip install tox
 tox

setup.py

@@ -53,7 +53,7 @@ def run(self):
 
 setup(
     name='deploy-config-generator',
-    version='2.29.0',
+    version='2.30.0',
     url='https://github.com/ApplauseOSS/deploy-config-generator',
     license='MIT',
     description='Utility to generate service deploy configurations',

tests/integration/age-key.txt

@@ -0,0 +1,6 @@
+# NOTICE: this key file was created for use in the integration tests and should not be
+# used elsewhere
+#
+# created: 2025-11-04T20:43:20Z
+# public key: age1cnlgtr6w8p8phdxqevklllex0u7y97574xclnf0qv050emsstgpqv23h0q
+AGE-SECRET-KEY-1D4HH6NNHZZSRM0WD23PEWT9FTWR2708FNDV957WPTJ0CE0WDFG9QY3QUSY

tests/integration/secrets/deploy/config.yml

@@ -0,0 +1,6 @@
+---
+test:
+  dummy: True
+  parent1:
+    - child2_1: '{{ SECRETS.foo }}'
+      child2_2: '{{ SECRETS.some }}'

tests/integration/secrets/expected_output/dummy-001.foo

@@ -0,0 +1,14 @@
+Dummy output plugin
+
+App config:
+
+{
+  "dummy": true,
+  "parent1": [
+    {
+      "child2_1": "bar",
+      "child2_2": "value",
+      "parent2": []
+    }
+  ]
+}

tests/integration/secrets/runme.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+TEST_DIR=$(dirname $0)
+cd ${TEST_DIR}
+
+export PYTHONPATH=../../..
+
+export SOPS_AGE_KEY_FILE=../age-key.txt
+
+set -e
+
+rm -rf tmp
+mkdir tmp
+
+set -x
+
+python3 -m deploy_config_generator -v -c site_config.yml -o tmp . $@
+
+diff -BurN expected_output tmp

tests/integration/secrets/secrets.yaml

@@ -0,0 +1,17 @@
+some: ENC[AES256_GCM,data:x09vw1s=,iv:jbrwDiofbm02yu3g53EYrEvoytv2kwFNqc+r2DVfFQs=,tag:ovZMpmsGP7GRGj76zscP5w==,type:str]
+foo: ENC[AES256_GCM,data:kcOd,iv:ShGUV7ZGtRtw3RTicQT9kXyH0gwJHYAsiHx2N+/WPiA=,tag:oU13FMj1IdSv2wJLAmqJWA==,type:str]
+sops:
+    age:
+        - recipient: age1cnlgtr6w8p8phdxqevklllex0u7y97574xclnf0qv050emsstgpqv23h0q
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZnhsUzRTdlBGeEhEU2Rr
+            Y1JveWp2Mk9nMzQ2dGNNcXFjMWRlRWxKVHc0CitFdnNLZndCMjRRNS8rNDZuR2F5
+            VEFZa25Dd0JLVlp2cGJrZzhRZ014NUUKLS0tIE1lclE1UkprT1dkZGlNVzk2SUdn
+            eTdaRXl1aXNjN0hMZHFXR3FRYTJQcVEK0Wh7ZEGptMNgkzdChP0rAi4ce+Cf7ADR
+            XwmIQfyTBk8CiRu8fGzNOfpN2WrDXbaOYWUAphUUDbWs+/zdFnQfKg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2025-11-04T20:44:45Z"
+    mac: ENC[AES256_GCM,data:shNpmymZcVo08ydUtVNMP6KNry9elfSsr3U6WVckNhYTdLtBt5Hxva78o1eNc1dslKZHji8xTTooV4umi/0k9+Cos62jP6snozPhqsxY7QpXx7BzlFWXE1cVwRndIPZ9S70ylXmO2HEznzOsGsQ8z0DI5l8rSMucwO0Yg8Q+ltY=,iv:Ds0WOxw3fqjYKpGqDRKz6BOmv4hyNcfHMx8FJU3UW+A=,tag:P97g2dsaNUYezp+EDWEyZQ==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.11.0

tests/integration/secrets/site_config.yml

@@ -0,0 +1,10 @@
+---
+default_output: dummy
+plugins:
+  dummy:
+    # Enable 'dummy' plugin, which is disabled by default
+    enabled: true
+secrets_files:
+  - secrets.yaml
+  # This is here to make sure we don't blow up when a secrets file doesn't exist
+  - secrets.doesnotexist.yaml