Merge pull request #157 from puppetlabs/transport

(FM-7695) Transports - the remote content framework

Author: da-ar

.rubocop.yml

@@ -1,6 +1,5 @@
 ---
 require: rubocop-rspec
-inherit_from: .rubocop_todo.yml
 AllCops:
   TargetRubyVersion: '2.1'
   Include:
@@ -157,4 +156,4 @@ Naming/UncommunicativeMethodParamName:
 
 # This cop breaks syntax highlighting in VSCode
 Layout/ClosingHeredocIndentation:
-  Enabled: false
+  Enabled: false

.rubocop_todo.yml

@@ -25,7 +25,7 @@ matrix:
     - env: RVM="jruby-9.1.9.0" PUPPET_GEM_VERSION='~> 5' JRUBY_OPTS="--debug"
       before_cache: pushd ~/.rvm && rm -rf archives rubies/ruby-2.2.7 rubies/ruby-2.3.4 && popd
       cache:
-        bundler: true
+        bundler: false
         directories: ~/.rvm
       before_install: rvm use jruby-9.1.9.0 --install --binary --fuzzy
     - rvm: 2.4.3
@@ -65,10 +65,5 @@ matrix:
     - rvm: 2.5.1
       env: PUPPET_GEM_VERSION='https://github.com/puppetlabs/puppet.git#6.0.x'
 notifications:
-  hipchat:
-    rooms:
-      secure: 10a49kkZcghKHNnef8x7eBG+KjScL3i1VpygFg6DPAOK2YNbEoyEx1Kv9KLC7GSRYov/SQZOsZrvHZtDhEtFSKhhiAjOwxl1jV1t6aAMGMnN1IoZBOvdAJKrZsm54/bBeYp+je2wqnnoFNtLVFSoOX0LkFaDEWT+zGZ5xKJIH25GpeQEZf1eDxs/d8YX/m+RwbGXHVA//hOpvZo0ntvznh2EbW5OPODKSeUXbWZ+W4ndODTsKWFc/WLMSSgFDzW/Y2/9V40D4IC8lvSx6eKFryMfAQy6pO/d1aTB468awzyVcdYAMMCOITm7hlKGRKxNgq6NkOsXs5KLg6ifpn+a/Rhapbz6Qxbpjjho/7Wxngl4B3T+i35ap/mFrS/fOfKCq3gEQlYn29its9bEFArNGbr+/sXKABb+sRpgW4RTPWYDHJyHJendbevd5tZ+fd0JUBOi0Cb4PcXxQxM8IQrbuu2zso0K5MV05kL0S1DE/VsuUrPaK0RsF+b1+i6NfvtN8kgbYs1eiVku+guIG2ec3xIefQ1hsEOFPFNqSDfHp7nANnRVIbBCt8qw8DhmNEczsfN5Dp21euJUsO9qpau++NzD3jRhkE5Zki5cwsakU7hIQzw82BIb0eSQJCHygieExeEboWRqtDgy/IKIWPgIvEuU68ppR2bl2reKCHLCnWc=
-    template:
-    - '%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}
-      (<a href="%{build_url}">Details</a> | <a href="%{compare_url}">PR/Diff</a>)'
-    format: html
+  slack:
+    secure: aPXZYNow8LsmmlS8PQU3FjL0bc7FqUUA95d++wZfIu7YAjGboIUiekxYouQ0XnY+Aig8InvbTOIgBHgGNheyr/YFbFS90/jtulbF8oW7BitW+imgjeAHDCwlQZTCc4FFYde/2pI7QTT8H5NpLR9mKxlTU77Sqr8gFAIybuPdHcKMYQZdEZS07ma2pUp7+GyKS6PDQpzW2+mDCz/wfi3/JdsUvc0mclCZ8vxySc66j5P1E6nFDMzuakBOjwJHpgeDpreapbmSUQLAX0a3ZsFP+N+SNduLotlV2BWnJK2gcO6rGFP4Fz1D0bGXuBnYYdIiB+9OgI3wtXg9y1SifNHUG3IrOBAA8CGNyrebTGKtH0TS2O+HZLbaNX2g6udD5e3156vys9wScmJuQ/rSkVtQfXf1qUm5eijvlXI+DIbssbZHqm6QQGyM4p3NoULmNmF1C85bQoZ4GF7b1P/8mstsVE/HUfnzRPNbwD0r6j1aE/ck3PKMi7ZAhIi0Ja9RnAgP3wi0t62uERYcJGGYEycWohMWnrf2w6GFwGeuoiwAkASdHOLX0/AOMPc4mBOjlc621o8uYMrrZqfF5CrOAvJ151USSsWn2AhXaibIvnHo6X91paNvvNpU/GYu3CUAl6q8OhYovvjtRVPVnhs2DrpgoRB+6NWHnzjRG/wr6Z9U+vA=

.travis.yml

@@ -5,12 +5,15 @@
 require 'puppet/resource_api/property'
 require 'puppet/resource_api/puppet_context' unless RUBY_PLATFORM == 'java'
 require 'puppet/resource_api/read_only_parameter'
+require 'puppet/resource_api/transport'
+require 'puppet/resource_api/transport/wrapper'
 require 'puppet/resource_api/type_definition'
 require 'puppet/resource_api/value_creator'
 require 'puppet/resource_api/version'
 require 'puppet/type'
 require 'puppet/util/network_device'
 
+# This module contains the main API to register and access types, providers and transports.
 module Puppet::ResourceApi
   @warning_count = 0
 
@@ -38,7 +41,12 @@ def register_type(definition)
       # Keeps a copy of the provider around. Weird naming to avoid clashes with puppet's own `provider` member
       define_singleton_method(:my_provider) do
         @my_provider ||= Hash.new { |hash, key| hash[key] = Puppet::ResourceApi.load_provider(definition[:name]).new }
-        @my_provider[Puppet::Util::NetworkDevice.current.class]
+
+        if Puppet::Util::NetworkDevice.current.is_a? Puppet::ResourceApi::Transport::Wrapper
+          @my_provider[Puppet::Util::NetworkDevice.current.transport.class]
+        else
+          @my_provider[Puppet::Util::NetworkDevice.current.class]
+        end
       end
 
       # make the provider available in the instance's namespace
@@ -352,7 +360,7 @@ def cache_current_state(resource_hash)
       end
 
       define_singleton_method(:context) do
-        @context ||= PuppetContext.new(definition)
+        @context ||= PuppetContext.new(TypeDefinition.new(definition))
       end
 
       def context
@@ -410,8 +418,10 @@ def load_provider(type_name)
     type_name_sym = type_name.to_sym
     device_name = if Puppet::Util::NetworkDevice.current.nil?
                     nil
-                  else
+                  elsif Puppet::Util::NetworkDevice.current.is_a? Puppet::ResourceApi::Transport::Wrapper
                     # extract the device type from the currently loaded device's class
+                    Puppet::Util::NetworkDevice.current.schema.name
+                  else
                     Puppet::Util::NetworkDevice.current.class.name.split('::')[-2].downcase
                   end
     device_class_name = class_name_from_type_name(device_name)
@@ -451,6 +461,12 @@ def load_device_provider(class_name, type_name_sym, device_class_name, device_na
   end
   module_function :load_device_provider # rubocop:disable Style/AccessModifierDeclarations
 
+  # keeps the existing register API format. e.g. Puppet::ResourceApi.register_type
+  def register_transport(schema)
+    Puppet::ResourceApi::Transport.register(schema)
+  end
+  module_function :register_transport # rubocop:disable Style/AccessModifierDeclarations
+
   def self.class_name_from_type_name(type_name)
     type_name.to_s.split('_').map(&:capitalize).join
   end

lib/puppet/resource_api.rb

@@ -1,20 +1,34 @@
 require 'puppet/resource_api/type_definition'
 
+# rubocop:disable Style/Documentation
 module Puppet; end
 module Puppet::ResourceApi; end
+# rubocop:enable Style/Documentation
+
+# This class provides access to all common external dependencies of providers and transports.
+# The runtime environment will inject an appropriate implementation.
 class Puppet::ResourceApi::BaseContext
   attr_reader :type
 
   def initialize(definition)
-    raise ArgumentError, 'BaseContext requires definition to be a Hash' unless definition.is_a?(Hash)
-    @typename = definition[:name]
-    @type = Puppet::ResourceApi::TypeDefinition.new(definition)
+    if definition.is_a?(Hash)
+      # this is only for backwards compatibility
+      @type = Puppet::ResourceApi::TypeDefinition.new(definition)
+    elsif definition.is_a? Puppet::ResourceApi::BaseTypeDefinition
+      @type = definition
+    else
+      raise ArgumentError, 'BaseContext requires definition to be a child of Puppet::ResourceApi::BaseTypeDefinition, not <%{actual_type}>' % { actual_type: definition.class }
+    end
   end
 
   def device
     raise 'Received device() on an unprepared BaseContext. Use a PuppetContext instead.'
   end
 
+  def transport
+    raise 'No transport available.'
+  end
+
   def failed?
     @failed
   end
@@ -27,7 +41,7 @@ def feature_support?(feature)
   [:debug, :info, :notice, :warning, :err].each do |level|
     define_method(level) do |*args|
       if args.length == 1
-        message = "#{@context || @typename}: #{args.last}"
+        message = "#{@context || @type.name}: #{args.last}"
       elsif args.length == 2
         resources = format_titles(args.first)
         message = "#{resources}: #{args.last}"
@@ -137,9 +151,9 @@ def send_log(_level, _message)
 
   def format_titles(titles)
     if titles.length.zero? && !titles.is_a?(String)
-      @typename
+      @type.name
     else
-      "#{@typename}[#{[titles].flatten.compact.join(', ')}]"
+      "#{@type.name}[#{[titles].flatten.compact.join(', ')}]"
     end
   end
 

lib/puppet/resource_api/base_context.rb

@@ -1,9 +1,14 @@
 require 'puppet/resource_api/base_context'
 
+# Implement Resource API Conext to log through an IO object, defaulting to `$stderr`.
+# There is no access to a device here. You can supply a transport if necessary.
 class Puppet::ResourceApi::IOContext < Puppet::ResourceApi::BaseContext
-  def initialize(definition, target = $stderr)
+  attr_reader :transport
+
+  def initialize(definition, target = $stderr, transport = nil)
     super(definition)
     @target = target
+    @transport = transport
   end
 
   protected

lib/puppet/resource_api/io_context.rb

@@ -1,13 +1,19 @@
 require 'puppet/resource_api/base_context'
 require 'puppet/util/logging'
 
+# Implement Resource API Context to log through Puppet facilities
+# and access/expose the puppet process' current device/transport
 class Puppet::ResourceApi::PuppetContext < Puppet::ResourceApi::BaseContext
   def device
     # TODO: evaluate facter_url setting for loading config if there is no `current` NetworkDevice
     raise 'no device configured' unless Puppet::Util::NetworkDevice.current
     Puppet::Util::NetworkDevice.current
   end
 
+  def transport
+    device.transport
+  end
+
   def log_exception(exception, message: 'Error encountered', trace: false)
     super(exception, message: message, trace: trace || Puppet[:trace])
   end

lib/puppet/resource_api/puppet_context.rb

@@ -0,0 +1,95 @@
+module Puppet::ResourceApi; end # rubocop:disable Style/Documentation
+
+# Remote target transport API
+module Puppet::ResourceApi::Transport
+  def register(schema)
+    raise Puppet::DevError, 'requires a hash as schema, not `%{other_type}`' % { other_type: schema.class } unless schema.is_a? Hash
+    raise Puppet::DevError, 'requires a `:name`' unless schema.key? :name
+    raise Puppet::DevError, 'requires `:desc`' unless schema.key? :desc
+    raise Puppet::DevError, 'requires `:connection_info`' unless schema.key? :connection_info
+    raise Puppet::DevError, '`:connection_info` must be a hash, not `%{other_type}`' % { other_type: schema[:connection_info].class } unless schema[:connection_info].is_a?(Hash)
+
+    init_transports
+    unless @transports[@environment][schema[:name]].nil?
+      raise Puppet::DevError, 'Transport `%{name}` is already registered for `%{environment}`' % {
+        name: schema[:name],
+        environment: @environment,
+      }
+    end
+    @transports[@environment][schema[:name]] = Puppet::ResourceApi::TransportSchemaDef.new(schema)
+  end
+  module_function :register # rubocop:disable Style/AccessModifierDeclarations
+
+  # retrieve a Hash of transport schemas, keyed by their name.
+  def list
+    init_transports
+    Marshal.load(Marshal.dump(@transports[@environment]))
+  end
+  module_function :list # rubocop:disable Style/AccessModifierDeclarations
+
+  def connect(name, connection_info)
+    validate(name, connection_info)
+    require "puppet/transport/#{name}"
+    class_name = name.split('_').map { |e| e.capitalize }.join
+    Puppet::Transport.const_get(class_name).new(get_context(name), wrap_sensitive(name, connection_info))
+  end
+  module_function :connect # rubocop:disable Style/AccessModifierDeclarations
+
+  def inject_device(name, transport)
+    transport_wrapper = Puppet::ResourceApi::Transport::Wrapper.new(name, transport)
+
+    if Puppet::Util::NetworkDevice.respond_to?(:set_device)
+      Puppet::Util::NetworkDevice.set_device(name, transport_wrapper)
+    else
+      Puppet::Util::NetworkDevice.instance_variable_set(:@current, transport_wrapper)
+    end
+  end
+  module_function :inject_device # rubocop:disable Style/AccessModifierDeclarations
+
+  def self.validate(name, connection_info)
+    init_transports
+    require "puppet/transport/schema/#{name}" unless @transports[@environment].key? name
+    transport_schema = @transports[@environment][name]
+    if transport_schema.nil?
+      raise Puppet::DevError, 'Transport for `%{target}` not registered with `%{environment}`' % {
+        target: name,
+        environment: @environment,
+      }
+    end
+    message_prefix = 'The connection info provided does not match the Transport Schema'
+    transport_schema.check_schema(connection_info, message_prefix)
+    transport_schema.validate(connection_info)
+  end
+  private_class_method :validate
+
+  def self.get_context(name)
+    require 'puppet/resource_api/puppet_context'
+    Puppet::ResourceApi::PuppetContext.new(@transports[@environment][name])
+  end
+  private_class_method :get_context
+
+  def self.init_transports
+    lookup = Puppet.lookup(:current_environment) if Puppet.respond_to? :lookup
+    @environment =  if lookup.nil?
+                      :transports_default
+                    else
+                      lookup.name
+                    end
+    @transports ||= {}
+    @transports[@environment] ||= {}
+  end
+  private_class_method :init_transports
+
+  def self.wrap_sensitive(name, connection_info)
+    transport_schema = @transports[@environment][name]
+    if transport_schema
+      transport_schema.definition[:connection_info].each do |attr_name, options|
+        if options.key?(:sensitive) && (options[:sensitive] == true)
+          connection_info[attr_name] = Puppet::Pops::Types::PSensitiveType::Sensitive.new(connection_info[attr_name])
+        end
+      end
+    end
+    connection_info
+  end
+  private_class_method :wrap_sensitive
+end

lib/puppet/resource_api/transport.rb

@@ -0,0 +1,55 @@
+require 'puppet/resource_api/transport'
+require 'hocon'
+require 'hocon/config_syntax'
+
+# Puppet::ResourceApi::Transport::Wrapper` to interface between the Util::NetworkDevice
+class Puppet::ResourceApi::Transport::Wrapper
+  attr_reader :transport, :schema
+
+  def initialize(name, url_or_config_or_transport)
+    if url_or_config_or_transport.is_a? String
+      url = URI.parse(url_or_config_or_transport)
+      raise "Unexpected url '#{url_or_config_or_transport}' found. Only file:/// URLs for configuration supported at the moment." unless url.scheme == 'file'
+      raise "Trying to load config from '#{url.path}, but file does not exist." if url && !File.exist?(url.path)
+      config = self.class.deep_symbolize(Hocon.load(url.path, syntax: Hocon::ConfigSyntax::HOCON) || {})
+    elsif url_or_config_or_transport.is_a? Hash
+      config = url_or_config_or_transport
+    elsif transport_class?(name, url_or_config_or_transport)
+      @transport = url_or_config_or_transport
+    end
+
+    @transport ||= Puppet::ResourceApi::Transport.connect(name, config)
+    @schema = Puppet::ResourceApi::Transport.list[name]
+  end
+
+  def transport_class?(name, transport)
+    class_name = name.split('_').map { |e| e.capitalize }.join
+    expected = Puppet::Transport.const_get(class_name).to_s
+    expected == transport.class.to_s
+  end
+
+  def facts
+    context = Puppet::ResourceApi::PuppetContext.new(@schema)
+    # @transport.facts + custom_facts  # look into custom facts work by TP
+    @transport.facts(context)
+  end
+
+  def respond_to_missing?(name, _include_private)
+    (@transport.respond_to? name) || super
+  end
+
+  def method_missing(method_name, *args, &block)
+    if @transport.respond_to? method_name
+      @transport.send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+
+  # From https://stackoverflow.com/a/11788082/4918
+  def self.deep_symbolize(obj)
+    return obj.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = deep_symbolize(v); } if obj.is_a? Hash
+    return obj.each_with_object([]) { |v, memo| memo << deep_symbolize(v); } if obj.is_a? Array
+    obj
+  end
+end