Fix plugin installation from gemfile

Several things are fixed:
 * Don't re-install a plugin referenced from the gemfile with every
   call to `bundle install`
 * If the version of a plugin referenced in the gemfile conflicts
   with what's in the plugin index, _do_ re-install it
 * If plugins aren't installed yet, don't throw cryptic errors from
   commands that don't implicitly install gems, such as
   `bundle check` and `bundle info`. This also applies if the
   plugin index references a system gem, and that gem is removed.

This is all accomplished by actuallying including plugins as
regular dependencies in the Gemfile, so that they end up in the
lockfile, and then just using the regular lockfile with the
plugins-only pass of the gemfile in the Plugin infrastructure.
This also means that non-specific version constraints can be
used for plugins, and you can update them with
`bundle update <plugin>` just like any other gem.

Co-authored-by: Diogo Fernandes <diogofernandesop@gmail.com>

Author: ccutrer

Manifest.txt

@@ -161,6 +161,7 @@ bundler/lib/bundler/plugin.rb
 bundler/lib/bundler/plugin/api.rb
 bundler/lib/bundler/plugin/api/source.rb
 bundler/lib/bundler/plugin/dsl.rb
+bundler/lib/bundler/plugin/dummy_source.rb
 bundler/lib/bundler/plugin/events.rb
 bundler/lib/bundler/plugin/index.rb
 bundler/lib/bundler/plugin/installer.rb

bundler/lib/bundler/cli/install.rb

@@ -55,7 +55,7 @@ def run
           removed_message: "The --binstubs option have been removed in favor of `bundle binstubs --all`"
       end
 
-      Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.feature_flag.plugins?
+      Plugin.gemfile_install(Bundler.default_gemfile, Bundler.default_lockfile) if Bundler.feature_flag.plugins?
 
       definition = Bundler.definition
       definition.validate_runtime!

bundler/lib/bundler/cli/update.rb

@@ -15,8 +15,6 @@ def run
 
       Bundler.self_manager.update_bundler_and_restart_with_it_if_needed(update_bundler) if update_bundler
 
-      Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.feature_flag.plugins?
-
       sources = Array(options[:source])
       groups  = Array(options[:group]).map(&:to_sym)
 
@@ -33,29 +31,39 @@ def run
 
       conservative = options[:conservative]
 
-      if full_update
+      unlock = if full_update
         if conservative
-          Bundler.definition(conservative: conservative)
+          { conservative: conservative }
         else
-          Bundler.definition(true)
+          true
         end
       else
         unless Bundler.default_lockfile.exist?
           raise GemfileLockNotFound, "This Bundle hasn't been installed yet. " \
             "Run `bundle install` to update and install the bundled gems."
         end
-        Bundler::CLI::Common.ensure_all_gems_in_lockfile!(gems)
+        explicit_gems = gems
 
         if groups.any?
           deps = Bundler.definition.dependencies.select {|d| (d.groups & groups).any? }
           gems.concat(deps.map(&:name))
         end
 
-        Bundler.definition(gems: gems, sources: sources, ruby: options[:ruby],
-                           conservative: conservative,
-                           bundler: update_bundler)
+        {
+          gems: gems,
+          sources: sources,
+          ruby: options[:ruby],
+          conservative: conservative,
+          bundler: update_bundler,
+        }
       end
 
+      Plugin.gemfile_install(Bundler.default_gemfile, Bundler.default_lockfile, unlock.dup) if Bundler.feature_flag.plugins?
+
+      Bundler::CLI::Common.ensure_all_gems_in_lockfile!(explicit_gems) if explicit_gems
+
+      Bundler.definition(unlock)
+
       Bundler::CLI::Common.configure_gem_version_promoter(Bundler.definition, options)
 
       Bundler::Fetcher.disable_endpoint = options["full-index"]

bundler/lib/bundler/definition.rb

@@ -105,6 +105,8 @@ def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, opti
           @locked_specs   = SpecSet.new([])
           @locked_sources = []
         end
+
+        remove_plugin_dependencies_if_necessary
       else
         @unlock         = {}
         @platforms      = []
@@ -201,23 +203,20 @@ def removed_specs
       @locked_specs - specs
     end
 
+    def plugins
+      @plugins ||= resolve.materialize(plugin_dependencies)
+    end
+
+    def missing_plugins?
+      missing_specs_from_set? { plugins.missing_specs }
+    end
+
     def missing_specs
       resolve.materialize(requested_dependencies).missing_specs
     end
 
     def missing_specs?
-      missing = missing_specs
-      return false if missing.empty?
-      Bundler.ui.debug "The definition is missing #{missing.map(&:full_name)}"
-      true
-    rescue BundlerError => e
-      @resolve = nil
-      @resolver = nil
-      @resolution_packages = nil
-      @specs = nil
-
-      Bundler.ui.debug "The definition is missing dependencies, failed to resolve & materialize locally (#{e})"
-      true
+      missing_specs_from_set? { missing_specs }
     end
 
     def requested_specs
@@ -228,6 +227,10 @@ def requested_dependencies
       dependencies_for(requested_groups)
     end
 
+    def plugin_dependencies
+      requested_dependencies.select {|dep| dep.type == :plugin }
+    end
+
     def current_dependencies
       filter_relevant(dependencies)
     end
@@ -417,6 +420,7 @@ def ensure_equivalent_gemfile_and_lockfile(explicit_flag = false)
     def validate_runtime!
       validate_ruby!
       validate_platforms!
+      validate_plugins!
     end
 
     def validate_ruby!
@@ -452,6 +456,19 @@ def validate_platforms!
         "Add the current platform to the lockfile with\n`bundle lock --add-platform #{local_platform}` and try again."
     end
 
+    def validate_plugins!
+      missing_plugins_list = []
+      plugin_dependencies.each do |plugin|
+        missing_plugins_list << plugin unless Plugin.installed?(plugin.name)
+      end
+      missing_plugins_list.map! {|p| "#{p.name} (#{p.requirement})" }
+      if missing_plugins_list.size > 1
+        raise GemNotFound, "Plugins #{missing_plugins_list.join(", ")} are not installed"
+      elsif missing_plugins_list.any?
+        raise GemNotFound, "Plugin #{missing_plugins_list.join(", ")} is not installed"
+      end
+    end
+
     def add_platform(platform)
       @new_platform ||= !@platforms.include?(platform)
       @platforms |= [platform]
@@ -497,6 +514,21 @@ def unlocking?
 
     private
 
+    def missing_specs_from_set?
+      missing = yield
+      return false if missing.empty?
+      Bundler.ui.debug "The definition is missing #{missing.map(&:full_name)}"
+      true
+    rescue BundlerError => e
+      @resolve = nil
+      @resolver = nil
+      @resolution_packages = nil
+      @specs = nil
+
+      Bundler.ui.debug "The definition is missing dependencies, failed to resolve & materialize locally (#{e})"
+      true
+    end
+
     def should_add_extra_platforms?
       !lockfile_exists? && generic_local_platform_is_ruby? && !Bundler.settings[:force_ruby_platform]
     end
@@ -1067,5 +1099,14 @@ def remove_invalid_platforms!(dependencies)
     def source_map
       @source_map ||= SourceMap.new(sources, dependencies, @locked_specs)
     end
+
+    def remove_plugin_dependencies_if_necessary
+      return if Bundler.feature_flag.plugins_in_lockfile?
+      # we already have plugin dependencies in the lockfile; continue to do so regardless
+      # of the current setting
+      return if @dependencies.any? {|d| d.type == :plugin && @locked_deps.key?(d.name) }
+
+      @dependencies.reject! {|d| d.type == :plugin }
+    end
   end
 end

bundler/lib/bundler/dependency.rb

@@ -29,7 +29,15 @@ class Dependency < Gem::Dependency
 
     def initialize(name, version, options = {}, &blk)
       type = options["type"] || :runtime
-      super(name, version, type)
+      if type == :plugin
+        # RubyGems doesn't support plugin type, which only
+        # makes sense in the context of Bundler, so bypass
+        # the RubyGems validation
+        super(name, version, :runtime)
+        @type = type
+      else
+        super(name, version, type)
+      end
 
       @autorequire    = nil
       @groups         = Array(options["group"] || :default).map(&:to_sym)

bundler/lib/bundler/dsl.rb

@@ -160,9 +160,15 @@ def source(source, *args, &blk)
 
       if options.key?("type")
         options["type"] = options["type"].to_s
-        unless Plugin.source?(options["type"])
+        unless (source_plugin = Plugin.source_plugin(options["type"]))
           raise InvalidOption, "No plugin sources available for #{options["type"]}"
         end
+        # Implicitly add a dependency on source plugins who are named bundler-source-<type>,
+        # and aren't already mentioned in the Gemfile.
+        # See also Plugin::DSL#source
+        if source_plugin.start_with?("bundler-source-") && !@dependencies.any? {|d| d.name == source_plugin }
+          plugin(source_plugin)
+        end
 
         unless block_given?
           raise InvalidOption, "You need to pass a block to #source with :type option"
@@ -271,7 +277,13 @@ def env(name)
     end
 
     def plugin(*args)
-      # Pass on
+      options = args.last.is_a?(Hash) ? args.pop.dup : {}
+
+      normalize_hash(options)
+      options["type"] = :plugin
+      options["require"] = false
+
+      gem(*args, options)
     end
 
     def method_missing(name, *args)

bundler/lib/bundler/feature_flag.rb

@@ -35,6 +35,7 @@ def self.settings_method(name, key, &default)
     settings_flag(:global_gem_cache) { bundler_3_mode? }
     settings_flag(:path_relative_to_cwd) { bundler_3_mode? }
     settings_flag(:plugins) { @bundler_version >= Gem::Version.new("1.14") }
+    settings_flag(:plugins_in_lockfile) { bundler_3_mode? }
     settings_flag(:print_only_version_number) { bundler_3_mode? }
     settings_flag(:setup_makes_kernel_gem_public) { !bundler_3_mode? }
     settings_flag(:update_requires_all_flag) { bundler_4_mode? }

bundler/lib/bundler/man/bundle-config.1

@@ -163,6 +163,8 @@ The following is a list of all configuration keys and their purpose\. You can le
 .IP "\(bu" 4
 \fBplugins\fR (\fBBUNDLE_PLUGINS\fR): Enable Bundler's experimental plugin system\.
 .IP "\(bu" 4
+\fBplugins_in_lockfile\fR (\fBBUNDLE_PLUGINS_IN_LOCKFILE\fR): Include plugins as regular dependencies in the lockfile\.
+.IP "\(bu" 4
 \fBprefer_patch\fR (BUNDLE_PREFER_PATCH): Prefer updating only to next patch version during updates\. Makes \fBbundle update\fR calls equivalent to \fBbundler update \-\-patch\fR\.
 .IP "\(bu" 4
 \fBprint_only_version_number\fR (\fBBUNDLE_PRINT_ONLY_VERSION_NUMBER\fR): Print only version number from \fBbundler \-\-version\fR\.

bundler/lib/bundler/man/bundle-config.1.ronn

@@ -234,6 +234,8 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html).
    Makes `--path` relative to the CWD instead of the `Gemfile`.
 * `plugins` (`BUNDLE_PLUGINS`):
    Enable Bundler's experimental plugin system.
+* `plugins_in_lockfile` (`BUNDLE_PLUGINS_IN_LOCKFILE`):
+   Include plugins as regular dependencies in the lockfile.
 * `prefer_patch` (BUNDLE_PREFER_PATCH):
    Prefer updating only to next patch version during updates. Makes `bundle update` calls equivalent to `bundler update --patch`.
 * `print_only_version_number` (`BUNDLE_PRINT_ONLY_VERSION_NUMBER`):