12597 - Create Plugin Packaging

MCollective plugin application allows us to package mcollective plugins as
debs or rpms using the mco command. Plugin types are defined by definition
classes that populates a list of files and metadata which are in turn passed
over to packager classes which will build the action package.

Currently agent plugins are defined in AgentDefinition and debs and rpms are created with the
OSpackagePackager, which uses FPM to output the packages.

Additionally the info action will display plugin related metadata and a list of packages
that will be created from a target directory.

ext/debian/mcollective-client.install

@@ -2,3 +2,4 @@ usr/bin/mco usr/bin/
 usr/sbin/mc-* usr/sbin/
 etc/mcollective/client.cfg etc/mcollective
 usr/share/mcollective/plugins/mcollective/application usr/share/mcollective/plugins/mcollective
+usr/share/mcollective/plugins/mcollective/pluginpackager usr/share/mcollective/plugins/mcollective

ext/redhat/mcollective.spec

@@ -118,6 +118,7 @@ fi
 %config(noreplace)/etc/mcollective/client.cfg
 %config/etc/mcollective/rpc-help.erb
 /usr/libexec/mcollective/mcollective/application
+/usr/libexec/mcollective/mcollective/pluginpackager
 
 %files
 %doc COPYING

lib/mcollective.rb

@@ -58,6 +58,7 @@ class MsgDoesNotMatchRequestID < RuntimeError; end
   autoload :Applications, "mcollective/applications"
   autoload :Vendor, "mcollective/vendor"
   autoload :Shell, "mcollective/shell"
+  autoload :PluginPackager, "mcollective/pluginpackager"
 
   MCollective::Vendor.load_vendored
 

lib/mcollective/monkey_patches.rb

@@ -51,3 +51,54 @@ def in_groups_of(chunk_size, padded_with=nil, &block)
   end unless method_defined?(:in_groups_of)
 end
 
+class Dir
+  def self.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+    case prefix_suffix
+    when nil
+      prefix = "d"
+      suffix = ""
+    when String
+      prefix = prefix_suffix
+      suffix = ""
+    when Array
+      prefix = prefix_suffix[0]
+      suffix = prefix_suffix[1]
+    else
+      raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+    end
+    tmpdir ||= Dir.tmpdir
+    t = Time.now.strftime("%Y%m%d")
+    n = nil
+    begin
+      path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+      path << "-#{n}" if n
+      path << suffix
+      Dir.mkdir(path, 0700)
+    rescue Errno::EEXIST
+      n ||= 0
+      n += 1
+      retry
+    end
+
+    if block_given?
+      begin
+        yield path
+      ensure
+        FileUtils.remove_entry_secure path
+      end
+    else
+      path
+    end
+  end unless method_defined?(:mktmpdir)
+
+  def self.tmpdir
+    tmp = '.'
+    for dir in [ENV['TMPDIR'], ENV['TMP'], ENV['TEMP'], '/tmp']
+      if dir and stat = File.stat(dir) and stat.directory? and stat.writable?
+        tmp = dir
+        break
+      end rescue nil
+    end
+    File.expand_path(tmp)
+  end unless method_defined?(:tmpdir)
+end

lib/mcollective/pluginmanager.rb

@@ -99,6 +99,65 @@ def self.create_instance(klass)
       end
     end
 
+    # Finds plugins in all configured libdirs
+    #
+    #   find("agent")
+    #
+    # will return an array of just agent names, for example:
+    #
+    #   ["puppetd", "package"]
+    #
+    # Can also be used to find files of other extensions:
+    #
+    #   find("agent", "ddl")
+    #
+    # Will return the same list but only of files with extension .ddl
+    # in the agent subdirectory
+    def self.find(type, extension="rb")
+      extension = ".#{extension}" unless extension.match(/^\./)
+
+      plugins = []
+
+      Config.instance.libdir.each do |libdir|
+        plugdir = File.join([libdir, "mcollective", type])
+
+        Dir.new(plugdir).grep(/#{extension}$/).map do |plugin|
+          plugins << File.basename(plugin, extension)
+        end
+      end
+
+      plugins.sort.uniq
+    end
+
+    # Finds and loads from disk all plugins from all libdirs that match
+    # certain criteria.
+    #
+    #    find_and_load("pluginpackager")
+    #
+    # Will find all .rb files in the libdir/mcollective/pluginpackager/
+    # directory in all libdirs and load them from disk.
+    #
+    # You can influence what plugins get loaded using a block notation:
+    #
+    #    find_and_load("pluginpackager") do |plugin|
+    #       plugin.match(/puppet/)
+    #    end
+    #
+    # This will load only plugins matching /puppet/
+    def self.find_and_load(type, extension="rb")
+      extension = ".#{extension}" unless extension.match(/^\./)
+
+      klasses = find(type, extension).map do |plugin|
+        if block_given?
+          next unless yield(plugin)
+        end
+
+        "%s::%s::%s" % [ "MCollective", type.capitalize, plugin.capitalize ]
+      end.compact
+
+      klasses.sort.uniq.each {|klass| loadclass(klass, true)}
+    end
+
     # Loads a class from file by doing some simple search/replace
     # on class names and then doing a require.
     def self.loadclass(klass, squash_failures=false)

lib/mcollective/pluginpackager.rb

@@ -0,0 +1,15 @@
+module MCollective
+  module PluginPackager
+    # Plugin definition classes
+    autoload :AgentDefinition, "mcollective/pluginpackager/agent_definition"
+
+    # Package implementation plugins
+    def self.load_packagers
+      PluginManager.find_and_load("pluginpackager")
+    end
+
+    def self.[](klass)
+      const_get("#{klass}")
+    end
+  end
+end

lib/mcollective/pluginpackager/agent_definition.rb

@@ -0,0 +1,91 @@
+module MCollective
+  module PluginPackager
+    # MCollective Agent Plugin package
+    class AgentDefinition
+      attr_accessor :path, :packagedata, :metadata, :target_path, :vendor, :iteration, :postinstall
+
+      def initialize(path, name, vendor, postinstall, iteration)
+        @path = path
+        @packagedata = {}
+        @iteration = iteration || 1
+        @postinstall = postinstall
+        @vendor = vendor || "Puppet Labs"
+        @target_path = File.expand_path(@path)
+        @metadata = get_metadata
+        @metadata[:name] = name if name
+        @metadata[:name] = @metadata[:name].downcase.gsub(" ", "_")
+        identify_packages
+      end
+
+      # Identify present packages and populate packagedata hash.
+      def identify_packages
+        @packagedata[:common] = common
+        @packagedata[:agent] = agent
+        @packagedata[:client] = client
+      end
+
+      # Obtain Agent package files and dependencies.
+      def agent
+        agent = {:files => [],
+                 :dependencies => ["mcollective"],
+                 :description => "Agent plugin for #{@metadata[:name]}"}
+
+        agentdir = File.join(@path, "agent")
+
+        if check_dir agentdir
+          ddls = Dir.glob "#{agentdir}/*.ddl"
+          agent[:files] = (Dir.glob("#{agentdir}/*") - ddls)
+          implementations = Dir.glob("#{@metadata[:name]}/**")
+          agent[:files] += implementations unless implementations.empty?
+        else
+          return nil
+        end
+
+        agent[:dependencies] << "mcollective-#{@metadata[:name]}-common" if @packagedata[:common]
+        agent
+      end
+
+      # Obtain client package files and dependencies.
+      def client
+        client = {:files => [],
+                  :dependencies => ["mcollective-client"],
+                  :description => "Client plugin for #{@metadata[:name]}"}
+
+        clientdir = File.join(@path, "application")
+        bindir = File.join(@path, "bin")
+        ddldir = File.join(@path, "agent")
+
+        client[:files] += Dir.glob("#{clientdir}/*") if check_dir clientdir
+        client[:files] += Dir.glob("#{bindir}/*") if check_dir bindir
+        client[:files] += Dir.glob("#{ddldir}/*.ddl") if check_dir ddldir
+        client[:dependencies] << "mcollective-#{@metadata[:name]}-common" if @packagedata[:common]
+        client[:files].empty? ? nil : client
+      end
+
+      # Obtain common package files and dependencies.
+      def common
+        common = {:files =>[],
+                  :dependencies => ["mcollective-common"],
+                  :description => "Common libraries for #{@metadata[:name]}"}
+
+        commondir = File.join(@path, "util")
+        common[:files] += Dir.glob("#{commondir}/*") if check_dir commondir
+        common[:files].empty? ? nil : common
+      end
+
+      # Load plugin meta data from ddl file.
+      def get_metadata
+        ddl = MCollective::RPC::DDL.new("package", false)
+        ddl.instance_eval File.read(Dir.glob("#{@path}/agent/*.ddl").first)
+        ddl.meta
+      rescue
+        raise "error: could not read agent DDL File"
+      end
+
+      # Check if directory is present and not empty.
+      def check_dir(path)
+        (File.directory?(path) && !Dir.glob(path).empty?) ? true : false
+      end
+    end
+  end
+end

plugins/mcollective/application/plugin.rb

@@ -0,0 +1,130 @@
+module MCollective
+  class Application::Plugin<Application
+
+    exclude_argument_sections "common", "filter", "rpc"
+
+    description "MCollective Plugin Application"
+    usage <<-END_OF_USAGE
+mco plugin package [options] <directory>
+       mco plugin info <directory>
+       mco plugin doc <agent>
+
+          info : Display plugin information including package details.
+       package : Create all available plugin packages.
+           doc : Application list and RPC agent help
+    END_OF_USAGE
+
+    option  :pluginname,
+            :description => "Plugin name",
+            :arguments => ["-n", "--name NAME"],
+            :type => String
+
+    option :postinstall,
+           :description => "Post install script",
+           :arguments => ["--postinstall POSTINSTALL"],
+           :type => String
+
+    option :iteration,
+           :description => "Iteration number",
+           :arguments => ["--iteration ITERATION"],
+           :type => String
+
+    option :vendor,
+           :description => "Vendor name",
+           :arguments => ["--vendor VENDOR"],
+           :type => String
+
+    option :format,
+           :description => "Package output format. Defaults to rpm or deb",
+           :arguments => ["--format OUTPUTFORMAT"],
+           :type => String
+
+    option :plugintype,
+           :description => "Plugin type.",
+           :arguments => ["--plugintype PLUGINTYPE"],
+           :type => String
+
+    option :rpctemplate,
+           :description => "RPC Template to use.",
+           :arguments => ["--template RPCHELPTEMPLATE"],
+           :type => String
+
+    # Handle alternative format that optparser can't parse.
+    def post_option_parser(configuration)
+      if ARGV.length >= 1
+        configuration[:action] = ARGV.delete_at(0)
+
+        configuration[:target] = ARGV.delete_at(0) || "."
+      end
+    end
+
+    # Display info about plugin
+    def info_command
+      plugin = prepare_plugin
+      packager = PluginPackager["#{configuration[:format].capitalize}Packager"]
+      packager.new(plugin).package_information
+    end
+
+    # Package plugin
+    def package_command
+      plugin = prepare_plugin
+      packager = PluginPackager["#{configuration[:format].capitalize}Packager"]
+      packager.new(plugin).create_packages
+    end
+
+    # Show application list and RPC agent help
+    def doc_command
+      if configuration.include?(:target) && configuration[:target] != "."
+        ddl = MCollective::RPC::DDL.new(configuration[:target])
+        puts ddl.help(configuration[:rpctemplate] || Config.instance.rpchelptemplate)
+      else
+        puts "The Marionette Collective version #{MCollective.version}"
+        puts
+
+        PluginManager.find("agent", "ddl").each do |ddl|
+          help = MCollective::RPC::DDL.new(ddl)
+          puts "  %-15s %s" % [ddl, help.meta[:description]]
+        end
+      end
+    end
+
+    # Creates the correct package plugin object.
+    def prepare_plugin
+        set_plugin_type unless configuration[:plugintype]
+        configuration[:format] = "ospackage" unless configuration[:format]
+        PluginPackager.load_packagers
+        plugin_class = PluginPackager[configuration[:plugintype]]
+        plugin_class.new(configuration[:target], configuration[:pluginname], configuration[:vendor], configuration[:postinstall], configuration[:iteration])
+    end
+
+    def directory_for_type(type)
+      File.directory?(File.join(configuration[:target], type))
+    end
+
+    # Identify plugin type if not provided.
+    def set_plugin_type
+      if directory_for_type("agent") || directory_for_type("application")
+        configuration[:plugintype] = "AgentDefinition"
+      else
+        raise "error. target directory is not a valid mcollective plugin"
+      end
+    end
+
+    # Returns a list of available actions in a pretty format
+    def list_actions
+      methods.sort.grep(/_command/).map{|x| x.to_s.gsub("_command", "")}.join("|")
+    end
+
+    def main
+        abort "No action specified" unless configuration.include?(:action)
+
+        cmd = "#{configuration[:action]}_command"
+
+        if respond_to? cmd
+          send cmd
+        else
+          abort "Invalid action #{configuration[:action]}. Valid actions are [#{list_actions}]."
+        end
+    end
+  end
+end