Fixes #13667 - support for puppet 4.0 in puppet_proxy module

bundler.d/puppet.rb

@@ -1,4 +1,4 @@
-group :puppet, :puppetca do
+group :puppet_proxy_legacy do
   gem 'puppet', '< 5.0.0'
   gem 'ruby-augeas', :require => 'augeas'
 end

config/settings.d/puppet.yml.example

@@ -3,54 +3,12 @@
 :enabled: false
 
 # valid providers:
-#   puppetrun   (for puppetrun/kick, deprecated in Puppet 3)
-#   mcollective (uses mco puppet)
-#   puppetssh   (run puppet over ssh)
-#   salt        (uses salt puppet.run)
-#   customrun   (calls a custom command with args)
-#:puppet_provider: puppetrun
-
-# Custom salt puppet.run command
-# Set :salt_puppetrun_cmd to 'puppet.run agent no-noop' to run in no-noop mode.
-# Default command is puppet.run.
-#:salt_puppetrun_cmd: puppet.run
-
-# Customrun command details
-# Set :customrun_cmd to the full path of the script you want to run, instead of /bin/false
-:customrun_cmd: /bin/false
-# Set :customrun_args to any args you want to pass to your custom script. The hostname of the
-# system to run against will be appended after the custom commands.
-:customrun_args: -ay -f -s
-
-:puppet_conf: /etc/puppet/puppet.conf
-# whether to use sudo before the ssh command
-:puppetssh_sudo: false
-# the command which will be sent to the host
-:puppetssh_command: /usr/bin/puppet agent --onetime --no-usecacheonfailure
-# wait for the command to finish (and capture exit code), or detach process and return 0
-# Note: enabling this option causes the Foreman web UI to be blocked when executing puppetrun,
-# with timeout from the Browser and/or Foreman's REST client after 60 seconds.
-:puppetssh_wait: false
-# With which user should the proxy connect
-#:puppetssh_user: root
-#:puppetssh_keyfile: /etc/foreman-proxy/id_rsa
-
-# Which user to invoke sudo as to run puppet commands
-#:puppet_user: root
-
-# If you want to override the puppet_user above just for mco commands
-#:mcollective_user: peadmin
-
-# URL of the puppet master itself for API requests
-#:puppet_url: https://puppet.example.com:8140
-# SSL certificates used to access the puppet master API
-#:puppet_ssl_ca: /var/lib/puppet/ssl/certs/ca.pem
-#:puppet_ssl_cert: /var/lib/puppet/ssl/certs/puppet.example.com.pem
-#:puppet_ssl_key: /var/lib/puppet/ssl/private_keys/puppet.example.com.pem
-
-# Override use of Puppet's API to list environments, by default it will use only if
-# environmentpath is given in puppet.conf, else will look for environments in puppet.conf
-#:puppet_use_environment_api: true
-
-# Cache options
-#:use_cache: true
+#   puppet_proxy_puppetrun   (for puppetrun/kick, deprecated in Puppet 3)
+#   puppet_proxy_mcollective (uses mco puppet)
+#   puppet_proxy_puppetssh   (run puppet over ssh)
+#   puppet_proxy_salt        (uses salt puppet.run)
+#   puppet_proxy_customrun   (calls a custom command with args)
+#:use_provider: puppet_proxy_puppetrun
+
+# Puppet version used
+#:puppet_version: 4.1

config/settings.d/puppet_proxy_customrun.yml.example

@@ -0,0 +1,7 @@
+---
+# Set :command to the full path of the script you want to run, instead of /bin/false
+#:command: /bin/false
+#
+# Set :command_arguments to any args you want to pass to your custom script. The hostname of the
+# system to run against will be appended after the custom commands.
+#:command_arguments: -ay -f -s

config/settings.d/puppet_proxy_legacy.yml.example

@@ -0,0 +1,24 @@
+#
+# puppet_proxy_legacy module is used for puppet versions prior to 4.0
+#
+# puppet_proxy_legacy is configured automatcially based on
+# :puppet_version setting in smart-proxy's puppet.yml configuration file.
+#
+---
+#:puppet_conf: /etc/puppet/puppet.conf
+#
+# Override use of Puppet's API to list environments, by default it will use only if
+# environmentpath is given in puppet.conf, else will look for environments in puppet.conf
+# (Puppet versions prior to 4.0 only)
+#:use_environment_api: true
+#
+# URL of the puppet master itself for API requests. Required if puppet_use_environment_api is set to true.
+#:puppet_url: https://puppet.example.com:8140
+#
+# SSL certificates used to access the environment API. Required if puppet_use_environment_api is set to true.
+#:puppet_ssl_ca: /var/lib/puppet/ssl/certs/ca.pem
+#:puppet_ssl_cert: /var/lib/puppet/ssl/certs/puppet.example.com.pem
+#:puppet_ssl_key: /var/lib/puppet/ssl/private_keys/puppet.example.com.pem
+#
+# Enable/disable puppet class cache
+#:use_cache: true

config/settings.d/puppet_proxy_mcollective.yml.example

@@ -0,0 +1,12 @@
+---
+#
+# User for execution of mco commands
+#
+# sudo permission needs to be added to ensure
+# smart-proxy can execute 'sudo' command
+#
+# For Puppet Enterprise this means
+# Defaults:foreman-proxy !requiretty
+# foreman-proxy ALL=(peadmin) NOPASSWD: /opt/puppet/bin/mco *',
+#
+#:user: peadmin

config/settings.d/puppet_proxy_puppet_api.yml.example

@@ -0,0 +1,14 @@
+#
+# puppet_proxy_pupppet_api module is used for puppet versions 4.0 and higher
+#
+# puppet_proxy_pupppet_api is configured automatcially based on
+# :puppet_version setting in smart-proxy's puppet.yml configuration file.
+#
+---
+# URL of the puppet master itself for API requests.
+#:puppet_url: https://puppet.example.com:8140
+#
+# SSL certificates used to access the puppet API
+#:puppet_ssl_ca: /var/lib/puppet/ssl/certs/ca.pem
+#:puppet_ssl_cert: /var/lib/puppet/ssl/certs/puppet.example.com.pem
+#:puppet_ssl_key: /var/lib/puppet/ssl/private_keys/puppet.example.com.pem

config/settings.d/puppet_proxy_puppetrun.yml.example

@@ -0,0 +1,16 @@
+---
+#
+# User for execution of puppetrun commands
+#
+# sudo permission needs to be added to ensure
+# smart-proxy can execute 'sudo' command
+#
+# Defaults:foreman-proxy !requiretty
+# foreman-proxy ALL=(peadmin) NOPASSWD: /opt/puppet/bin/puppet *',
+#
+# or
+#
+# Defaults:foreman-proxy !requiretty
+# foreman-proxy ALL=(peadmin) NOPASSWD: /opt/puppet/bin/puppetrun *',
+#
+#:user: peadmin

config/settings.d/puppet_proxy_salt.yml.example

@@ -0,0 +1,3 @@
+---
+# Set :command to 'puppet.run agent no-noop' to run in no-noop mode. Default command is puppet.run.
+#:command: puppet.run

config/settings.d/puppet_proxy_ssh.yml.example

@@ -0,0 +1,15 @@
+---
+# the command which will be sent to the host
+#:command: puppet agent --onetime --no-usecacheonfailure
+#
+# whether to use sudo before the ssh command
+#:use_sudo: false
+#
+# wait for the command to finish (and capture exit code), or detach process and return 0
+# Note: enabling this option causes the Foreman web UI to be blocked when executing puppetrun,
+# with timeout from the Browser and/or Foreman's REST client after 60 seconds.
+#:wait: false
+#
+# With which user should the proxy connect
+#:user: root
+#:keyfile: /etc/foreman-proxy/id_rsa

extra/migrate_settings.rb

@@ -96,6 +96,10 @@ def underscore(src)
       src = src.tr("-", "_")
       src.downcase
     end
+
+    def strip_ruby_symbol_encoding(astring)
+      astring.gsub("!ruby/symbol ", ":").gsub("!ruby/sym ", ":")
+    end
   end
 
   class Migrator

extra/migrations/20150826000000_migrate_dhcp_settings.rb

@@ -62,8 +62,4 @@ def write_to_files(output)
       end
     end
   end
-
-  def strip_ruby_symbol_encoding(astring)
-    astring.gsub("!ruby/symbol ", ":").gsub("!ruby/sym ", ":")
-  end
 end

extra/migrations/20160413000000_migrate_puppet_settings.rb

@@ -0,0 +1,100 @@
+require 'yaml'
+
+class MigratePuppetSettings < ::Proxy::Migration
+  KNOWN_PARAMETERS = {
+      :enabled => [:puppet, :enabled],
+      :puppet_provider => [:puppet, :use_provider],
+      :puppet_user => [:puppet_proxy_puppetrun, :puppet_proxy_mcollective, :puppet_user],
+      :salt_puppetrun_cmd => [:puppet_proxy_salt, :command],
+      :customrun_cmd => [:puppet_proxy_customrun, :command],
+      :customrun_args => [:puppet_proxy_customrun, :command_arguments],
+      :use_cache => [:puppet_proxy_legacy, :use_cache],
+      :puppet_conf => [:puppet_proxy_legacy, :puppet_conf],
+      :puppet_use_environment_api => [:puppet_proxy_legacy, :use_environment_api],
+      :puppet_url => [:puppet_proxy_legacy, :puppet_proxy_puppet_api, :puppet_url],
+      :puppet_ssl_ca => [:puppet_proxy_legacy, :puppet_proxy_puppet_api, :puppet_ssl_ca],
+      :puppet_ssl_cert => [:puppet_proxy_legacy, :puppet_proxy_puppet_api, :puppet_ssl_cert],
+      :puppet_ssl_key => [:puppet_proxy_legacy, :puppet_proxy_puppet_api, :puppet_ssl_key],
+      :puppetssh_sudo => [:puppet_proxy_ssh, :use_sudo],
+      :puppetssh_command => [:puppet_proxy_ssh, :command],
+      :puppetssh_wait => [:puppet_proxy_ssh, :wait],
+      :puppetssh_user => [:puppet_proxy_ssh, :user],
+      :puppetssh_keyfile => [:puppet_proxy_ssh, :keyfile],
+      :mcollective_user => [:puppet_proxy_mcollective, :user]
+  }
+
+  def migrate
+    puppet_config = path(src_dir, "settings.d", "puppet.yml")
+    if !File.exist?(puppet_config)
+      duplicate_original_configuration
+      return
+    end
+
+    to_migrate = YAML.load_file(puppet_config)
+
+    output = migrate_puppet_configuration(to_migrate)
+    copy_original_configuration_except(path("settings.d", "puppet.yml"))
+    write_to_files(output)
+  end
+
+  def remap_parameter(aparameter, avalue)
+    module_names_to_parameter = KNOWN_PARAMETERS.has_key?(aparameter) ? KNOWN_PARAMETERS[aparameter] : [:unknown, aparameter]
+    parameter_name = module_names_to_parameter.last
+    module_names = module_names_to_parameter[0..-2]
+
+    avalue = old_provider_name_to_new(avalue) if parameter_name == :use_provider
+    module_names.map {|module_name| [module_name, parameter_name, avalue]}
+  end
+
+  def old_provider_name_to_new(aname)
+    if ['puppetrun', 'mcollective', 'puppetssh', 'salt', 'customrun'].include?(aname)
+      aname == 'puppetssh' ? 'puppet_proxy_ssh' : 'puppet_proxy_' + aname
+    else
+      aname
+    end
+  end
+
+  def puppet_version
+    require 'puppet'
+    Puppet::PUPPETVERSION
+  rescue Exception
+    "4.3.1"
+  end
+
+  def migrate_puppet_configuration(to_migrate)
+    migrated = Hash.new { |h,k| h[k] = Hash.new }
+    to_migrate.each do |option, value|
+      remap_parameter(option, value).each {|module_name, parameter_name, parameter_value| migrated[module_name][parameter_name] = parameter_value}
+    end
+
+    # deal with puppet_user setting, which used to be global, but has been moved (and renamed) to puppetrun and mcollective modules
+    if migrated.has_key?(:puppet_proxy_puppetrun)
+      puppetrun_user = migrated[:puppet_proxy_puppetrun].delete(:puppet_user)
+      migrated[:puppet_proxy_puppetrun][:user] = puppetrun_user unless puppetrun_user.nil?
+    end
+
+    if migrated.has_key?(:puppet_proxy_mcollective)
+      puppet_user = migrated[:puppet_proxy_mcollective].delete(:puppet_user)
+      if !(migrated[:puppet_proxy_mcollective].has_key?(:user) || puppet_user.nil?)
+        migrated[:puppet_proxy_mcollective][:user] = puppet_user
+      end
+    end
+
+    migrated[:puppet][:puppet_version] = puppet_version
+
+    migrated
+  end
+
+  def write_to_files(output)
+    output.keys.each do |m|
+      next if output[m].empty? || m == :unknown
+      File.open(path(dst_dir, "settings.d", "#{m}.yml"),'w') do |f|
+        f.write(strip_ruby_symbol_encoding(output[m].to_yaml))
+        if (m == :puppet) && !output[:unknown].empty?
+          f.write "\n# Unparsed options, please review\n"
+          f.write(strip_ruby_symbol_encoding(output[:unknown].to_yaml).gsub(/^---/,''))
+        end
+      end
+    end
+  end
+end

lib/proxy/error.rb

@@ -1,3 +1,16 @@
 module Proxy::Error
+  class HttpError < StandardError
+    attr_reader :status_code, :response_body
+    def initialize(status_code, response_body, msg = nil)
+      @status_code = status_code
+      @response_body = response_body
+      @msg = msg
+    end
+
+    def to_s
+      @msg.nil? ? "#{status_code} #{response_body}" : "#{@msg}: #{status_code} #{response_body}"
+    end
+  end
+
   class ConfigurationError < StandardError; end
 end

lib/proxy/plugin_initializer.rb

@@ -24,9 +24,13 @@ def resolve_providers(all_plugins_and_providers)
     providers = all_plugins_and_providers.select {|p| used_providers.include?(p[:name].to_sym)}
 
     not_available = used_providers - providers.map {|p| p[:name].to_sym}
-    return @providers = providers.map {|p| p[:class]} if not_available.empty?
 
-    fail_group_with_message("Disabling all modules in the group '#{member_names.join(', ')}': following providers are not available #{not_available}")
+    if not_available.empty?
+      logger.debug "Providers #{printable_module_names(used_providers)} are going to be configured for '#{@plugin.plugin_name}'"
+      return @providers = providers.map {|p| p[:class]}
+    end
+
+    fail_group_with_message("Disabling all modules in the group #{printable_module_names(member_names)}: following providers are not available #{printable_module_names(not_available)}")
   end
 
   def members
@@ -37,6 +41,11 @@ def member_names
     members.map {|m| m.plugin_name }
   end
 
+  def printable_module_names(names)
+    printable = names.map {|name| "'#{name}'"}.join(", ")
+    "[#{printable}]"
+  end
+
   def load_plugin_settings
     plugin.module_loader_class.new(plugin, di_container).load_settings
   rescue Exception => e
@@ -46,12 +55,10 @@ def load_plugin_settings
   def load_provider_settings
     return if failed?
     providers.each do |p|
-      begin
         p.module_loader_class.new(p, di_container).load_settings(plugin.settings.marshal_dump)
-      rescue Exception => e
-        fail_group(e)
-      end
     end
+  rescue Exception => e
+    fail_group(e)
   end
 
   def configure
@@ -64,7 +71,7 @@ def configure
   end
 
   def fail_group(an_exception)
-    fail_group_with_message("Disabling all modules in the group '#{member_names.join(', ')}' due to a failure in one of them: #{an_exception}", an_exception.backtrace)
+    fail_group_with_message("Disabling all modules in the group #{printable_module_names(member_names)} due to a failure in one of them: #{an_exception}", an_exception.backtrace)
   end
 
   def fail_group_with_message(a_message, a_backtrace = nil)
@@ -215,7 +222,7 @@ def merge_settings(provider_settings, main_plugin_settings)
     main_plugin_settings.delete(:enabled)
     # all modules have 'enabled' setting, we ignore it when looking for duplicate setting names
     if !(overlap = main_plugin_settings.keys - (main_plugin_settings.keys - provider_settings.keys)).empty?
-      raise "Provider '#{plugin.plugin_name}' settings conflict with the main plugin's settings: #{overlap}"
+      raise Exception, "Provider '#{plugin.plugin_name}' settings conflict with the main plugin's settings: #{overlap}"
     end
     provider_settings.merge(main_plugin_settings)
   end
@@ -254,8 +261,8 @@ def execute_validators(validations, config)
 
     validations.inject([]) do |all, validator|
       validator_class = available_validators[validator[:name]]
-      raise "Found an unknown validator '#{validator[:name]}' when validating '#{plugin.plugin_name}' module." if validator_class.nil?
-      validator_class.new(plugin, validator[:setting], validator[:args], validator[:predicate]).validate!(config)
+      raise Exception, "Encountered an unknown validator '#{validator[:name]}' when validating '#{plugin.plugin_name}' module." if validator_class.nil?
+      validator_class.new(plugin, validator[:setting], validator[:args], validator[:predicate]).evaluate_predicate_and_validate!(config)
       all << {:class => validator_class, :setting => validator[:setting], :args => validator[:args], :predicate => validator[:predicate]}
     end
   end