Merge pull request #24 from tailored-automation/patroni_dcs_config

Add ability to configure Patroni DCS configs

.github/workflows/ci.yaml

@@ -60,7 +60,7 @@ jobs:
       matrix:
         set:
           - "centos-7"
-          - "centos-8"
+          - "rocky-8"
           - "debian-9"
           - "debian-10"
           - "ubuntu-1804"

README.md

@@ -164,6 +164,25 @@ node pgarb {
 }
 ```
 
+Some values such as the PostgreSQL `max_connections` require changes to the [DCS configuration](https://patroni.readthedocs.io/en/latest/dynamic_configuration.html).
+This example shows using the `patroni_dcs_config` type with an `Exec` that will restart the Patroni cluster.
+
+```puppet
+include patroni
+
+patroni_dcs_config { 'postgresql.parameters.max_connections':
+  value  => 200,
+  notify => Exec['patroni-restart-pending'],
+}
+
+exec { 'patroni-restart-pending':
+  path        => '/usr/bin:/bin:/usr/sbin:/sbin',
+  command     => "sleep 60 ; ${patroni::patronictl} -c ${patroni::config_path} restart ${patroni::scope} --pending --force",
+  refreshonly => true,
+  require     => Service['patroni'],
+}
+```
+
 ## Reference
 
 All of the Patroni settings I could find in the [Patroni Settings Documentation](https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst) are mapped to this module.

data/os/Debian.yaml

@@ -2,5 +2,4 @@
 patroni::config_path: '/etc/patroni/config.yml'
 patroni::install_dependencies:
   - gcc
-  - python3-psycopg2
 patroni::python_class_version: '3'

data/os/RedHat.yaml

@@ -2,4 +2,3 @@
 patroni::config_path: '/opt/app/patroni/etc/postgresql.yml'
 patroni::install_dependencies:
   - gcc
-  - python3-psycopg2

data/os/RedHat/8.yaml

@@ -1,3 +1,4 @@
 ---
+patroni::manage_postgresql_repo: false
 patroni::python_class_version: '3'
 patroni::python_venv_version: '3'

data/os/Ubuntu.yaml

@@ -2,5 +2,4 @@
 patroni::config_path: '/etc/patroni/config.yml'
 patroni::install_dependencies:
   - gcc
-  - python3-psycopg2
 patroni::python_class_version: '3'

lib/puppet/provider/patroni_dcs_config/patronictl.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'patronictl'))
+
+Puppet::Type.type(:patroni_dcs_config).provide(:patronictl, parent: Puppet::Provider::Patronictl) do
+  desc 'Provider for patroni_dcs_config using patronictl'
+
+  mk_resource_methods
+
+  defaultfor kernel: ['Linux']
+
+  def self.instances
+    configs = []
+    begin
+      output = patronictl(['show-config'])
+    rescue Exception => e # rubocop:disable RescueException
+      Puppet.err('Failed to fetch patronictl configurations')
+      puts e.backtrace
+    end
+    Puppet.debug("show-config output: #{output}")
+    data = YAML.safe_load(output)
+    flatten_hash(data).each_pair do |key, value|
+      config = {}
+      config[:name] = key
+      config[:value] = value
+      configs << new(config)
+    end
+    configs
+  end
+
+  def self.prefetch(resources)
+    configs = instances
+    resources.keys.each do |name|
+      if provider = configs.find { |c| c.name == name } # rubocop:disable AssignmentInCondition
+        resources[name].provider = provider
+      end
+    end
+  end
+
+  def initialize(value = {})
+    super(value)
+    @property_flush = {}
+  end
+
+  type_properties.each do |prop|
+    define_method "#{prop}=".to_sym do |value|
+      @property_flush[prop] = value
+    end
+  end
+
+  def edit_config(value)
+    patronictl(['edit-config', '--force', '--quiet', '-s', value])
+  rescue Exception => e # rubocop:disable RescueException
+    raise Puppet::Error, "patronictl edit-config for #{resource[:name]} failed\nError message: #{e.message}"
+  end
+
+  def create
+    edit_config("#{resource[:name]}=#{resource[:value]}")
+  end
+
+  def flush
+    edit_config("#{resource[:name]}=#{@property_flush[:value]}")
+  end
+end

lib/puppet/provider/patronictl.rb

@@ -0,0 +1,40 @@
+# Parent class for patronictl providers
+class Puppet::Provider::Patronictl < Puppet::Provider
+  initvars
+
+  class << self
+    attr_accessor :path
+    attr_accessor :config
+  end
+
+  def self.patronictl(args, options = {})
+    cmd = [@path] + ['-c', @config] + args
+    default_options = { combine: true, override_locale: false, custom_environment: { 'LC_ALL' => 'en_US.utf8' } }
+    ret = execute(cmd, default_options.merge(options))
+    ret
+  end
+
+  def patronictl(*args)
+    self.class.patronictl(*args)
+  end
+
+  def self.type_properties
+    resource_type.validproperties.reject { |p| p.to_sym == :ensure }
+  end
+
+  def type_properties
+    self.class.type_properties
+  end
+
+  def self.flatten_hash(hash)
+    hash.each_with_object({}) do |(k, v), h|
+      if v.is_a? Hash
+        flatten_hash(v).map do |h_k, h_v|
+          h["#{k}.#{h_k}"] = h_v
+        end
+      else
+        h[k] = v
+      end
+    end
+  end
+end

lib/puppet/type/patroni_dcs_config.rb

@@ -0,0 +1,21 @@
+Puppet::Type.newtype(:patroni_dcs_config) do
+  desc <<-DESC
+@summary Manages Patroni DCS configuration options
+@example Set PostgreSQL max connections
+  patroni_dcs_config { 'postgresql.params.max_connections':
+    value => '200',
+  }
+DESC
+
+  newparam(:name, namevar: true) do
+    desc 'The DCS configuration option name'
+  end
+
+  newproperty(:value) do
+    desc 'The value to assign the DCS configuration'
+  end
+
+  autorequire(:service) do
+    ['patroni']
+  end
+end

lib/puppet/type/patronictl_config.rb

@@ -0,0 +1,38 @@
+Puppet::Type.newtype(:patronictl_config) do
+  desc <<-DESC
+@summary Abstract type to configure other types
+**NOTE** This is a private type not intended to be used directly.
+DESC
+
+  newparam(:name, namevar: true) do
+    desc 'The name of the resource.'
+  end
+
+  newparam(:path) do
+    desc 'patronictl path'
+    defaultto('/opt/app/patroni/bin/patronictl')
+  end
+
+  newparam(:config) do
+    desc 'patronictl config'
+    defaultto('/opt/app/patroni/etc/postgresql.yml')
+  end
+
+  # First collect all types with patronictl provider that come from this module
+  # For each patronictl type, set the class variable 'chunk_size' used by
+  # each provider to list resources
+  # Return empty array since we are not actually generating resources
+  def generate
+    patronictl_types = []
+    Dir[File.join(File.dirname(__FILE__), '../provider/patroni_*/patronictl.rb')].each do |file|
+      type = File.basename(File.dirname(file))
+      patronictl_types << type.to_sym
+    end
+    patronictl_types.each do |type|
+      provider_class = Puppet::Type.type(type).provider(:patronictl)
+      provider_class.path = self[:path]
+      provider_class.config = self[:config]
+    end
+    []
+  end
+end

manifests/init.pp

@@ -412,6 +412,23 @@
       before  => Service['patroni'],
     }
 
+    package { 'patroni-postgresql-devel-package':
+      ensure  => present,
+      name    => $postgresql::params::devel_package_name,
+      require => $postgres_repo_require,
+      before  => Service['patroni'],
+    }
+    if $install_method == 'pip' {
+      Package['patroni-postgresql-devel-package'] -> Python::Pip['psycopg2']
+    }
+
+    if $facts['os']['family'] == 'RedHat' and $manage_postgresql_repo and $default_bin_dir != '/usr/bin' {
+      file { '/usr/bin/pg_config':
+        ensure => 'link',
+        target => "${default_bin_dir}/pg_config",
+      }
+    }
+
     exec { 'patroni-clear-datadir':
       path        => '/usr/bin:/bin',
       command     => "/bin/rm -rf ${default_data_dir}",
@@ -546,4 +563,10 @@
     name   => $service_name,
     enable => $service_enable,
   }
+
+  $patronictl = "${install_dir}/bin/patronictl"
+  patronictl_config { 'puppet':
+    path   => $patronictl,
+    config => $config_path,
+  }
 }

metadata.json

@@ -47,6 +47,12 @@
         "8"
       ]
     },
+    {
+      "operatingsystem": "Rocky",
+      "operatingsystemrelease": [
+        "8"
+      ]
+    },
     {
       "operatingsystem": "Debian",
       "operatingsystemrelease": [

spec/acceptance/nodesets/centos-8.yml → spec/acceptance/nodesets/rocky-8.yml

@@ -6,7 +6,7 @@ HOSTS:
       - patroni1
     platform: el-8-x86_64
     hypervisor: docker
-    image: centos:8
+    image: rockylinux/rockylinux:8
     docker_preserve_image: true
     docker_cmd:
       - '/usr/sbin/init'
@@ -21,7 +21,7 @@ HOSTS:
       - patroni2
     platform: el-8-x86_64
     hypervisor: docker
-    image: centos:8
+    image: rockylinux/rockylinux:8
     docker_preserve_image: true
     docker_cmd:
       - '/usr/sbin/init'