Feature: metadata dependencies are auto installed

.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+Gemfile.local

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2018-06-17 09:41:41 +0200 using RuboCop version 0.49.1.
+# on 2019-06-28 22:09:48 +0200 using RuboCop version 0.49.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -23,7 +23,7 @@ Lint/EndAlignment:
     - 'lib/puppetlabs_spec_helper/rake_tasks.rb'
     - 'spec/watchr.rb'
 
-# Offense count: 4
+# Offense count: 7
 Lint/HandleExceptions:
   Exclude:
     - 'lib/puppetlabs_spec_helper/puppet_spec_helper.rb'
@@ -43,17 +43,17 @@ RSpec/FilePath:
     - 'spec/unit/puppetlabs_spec_helper/tasks/beaker_spec.rb'
     - 'spec/unit/puppetlabs_spec_helper/tasks/fixtures_spec.rb'
 
-# Offense count: 3
+# Offense count: 7
 # Configuration parameters: AssignmentOnly.
 RSpec/InstanceVariable:
   Exclude:
-    - 'spec/acceptance/smoke_spec.rb'
+    - 'spec/acceptance/spec_spec.rb'
 
 # Offense count: 6
 RSpec/MultipleExpectations:
   Max: 3
 
-# Offense count: 13
+# Offense count: 16
 RSpec/NamedSubject:
   Exclude:
     - 'spec/unit/puppetlabs_spec_helper/puppetlabs_spec/puppet_internals_spec.rb'
@@ -65,9 +65,10 @@ RSpec/VerifiedDoubles:
   Exclude:
     - 'spec/unit/puppetlabs_spec_helper/puppetlabs_spec/puppet_internals_spec.rb'
 
-# Offense count: 2
+# Offense count: 3
 Security/Eval:
   Exclude:
+    - 'Gemfile'
     - 'lib/puppetlabs_spec_helper/tasks/fixtures.rb'
 
 # Offense count: 4
@@ -79,7 +80,7 @@ Style/ClassAndModuleChildren:
     - 'lib/puppetlabs_spec_helper/puppetlabs_spec/matchers.rb'
     - 'lib/puppetlabs_spec_helper/puppetlabs_spec/puppet_internals.rb'
 
-# Offense count: 11
+# Offense count: 14
 Style/Documentation:
   Exclude:
     - 'spec/**/*'
@@ -88,8 +89,8 @@ Style/Documentation:
     - 'lib/puppetlabs_spec_helper/puppetlabs_spec/matchers.rb'
     - 'lib/puppetlabs_spec_helper/puppetlabs_spec/puppet_internals.rb'
     - 'lib/puppetlabs_spec_helper/tasks/beaker.rb'
-    - 'lib/puppetlabs_spec_helper/tasks/fixtures.rb'
     - 'lib/puppetlabs_spec_helper/tasks/check_symlinks.rb'
+    - 'lib/puppetlabs_spec_helper/tasks/fixtures.rb'
 
 # Offense count: 1
 Style/DoubleNegation:
@@ -103,7 +104,7 @@ Style/GlobalVars:
     - 'lib/puppetlabs_spec_helper/puppet_spec_helper.rb'
     - 'lib/puppetlabs_spec_helper/puppetlabs_spec/files.rb'
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: MinBodyLength.
 Style/GuardClause:
   Exclude:

.travis.yml

@@ -30,6 +30,7 @@ matrix:
   - rvm: '2.0'
     env: PUPPET_GEM_VERSION='~> 3.0'
   - rvm: '1.9'
+    dist: trusty
     env: PUPPET_GEM_VERSION='~> 3.0'
 notifications:
   email: false

Gemfile

@@ -14,6 +14,9 @@ group :development do
     gem 'rubocop', '< 0.50'
     gem 'rubocop-rspec', '~> 1'
   end
+  if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.3.0')
+    gem 'pry-byebug'
+  end
 end
 
 # json_pure 2.0.2 added a requirement on ruby >= 2. We pin to json_pure 2.0.1

README.md

@@ -191,6 +191,14 @@ file named `.fixtures.yml` in the root of the project. You can specify a alterna
 You can use the `MODULE_WORKING_DIR` environment variable to specify a diffent location when installing module fixtures via the forge. By default the
 working directory is `<module directory>/spec/fixtures/work-dir`.
 
+*Puppet Labs Spec Helper* supports installting modules from:
+
+ * SCM repositories via `repositories` key,
+ * Forge repositories via `forge_modules` key,
+ * Forge repositories from dependencies defined in `metadata.json` via `metadata` key.
+
+For more details on how to use those setting see [examples](#fixtures-examples).
+
 When specifying the repo source of the fixture you have a few options as to which revision of the codebase you wish to use, and optionally, the puppet versions where the fixture is needed.
 
  * `repo` - the url to the repo
@@ -302,6 +310,15 @@ fixtures:
       ref: "2.6.0"
 ```
 
+Install modules based on dependencies from `metadata.json`:
+
+```yaml
+fixtures:
+  metadata:
+    autoinstall_dependencies: true
+    forge: https://puppetforge.acmecorp.lan # optional
+```
+
 Pass additional flags to module installation:
 
 ```yaml
@@ -310,29 +327,29 @@ fixtures:
     stdlib:
       repo: "puppetlabs/stdlib"
       ref: "2.6.0"
-      flags: "--module_repository https://my_repo.com"
-    repositories:
-      firewall:
-        repo: "git://github.com/puppetlabs/puppetlabs-firewall"
-        ref: "2.6.0"
-        flags: "--verbose"
+      flags: "--module_repository https://puppetforge.acmecorp.lan"
+  repositories:
+    firewall:
+      repo: "git://github.com/puppetlabs/puppetlabs-firewall"
+      ref: "2.6.0"
+      flags: "--verbose"
 ```
 
 Use `defaults` to define global parameters:
 
 ```yaml
 defaults:
   forge_modules:
-    flags: "--module_repository https://my_repo.com"
+    flags: "--module_repository https://puppetforge.acmecorp.lan"
 fixtures:
   forge_modules:
     stdlib:
       repo: "puppetlabs/stdlib"
       ref: "2.6.0"
-    repositories:
-      firewall:
-        repo: "git://github.com/puppetlabs/puppetlabs-firewall"
-        ref: "2.6.0"
+  repositories:
+    firewall:
+      repo: "git://github.com/puppetlabs/puppetlabs-firewall"
+      ref: "2.6.0"
 ```
 
 Testing Parser Functions
@@ -389,7 +406,7 @@ environment variable``TEST_TIERS=high,medium``
 
 By default ``TEST_TIERS`` only accepts low, medium and high as valid tiers.  If you would like to use your own keywords to set the environment variable ``TEST_TIERS_ALLOWED``.
 
-For example: to use the keywords dev, rnd, staging and production you can set 
+For example: to use the keywords dev, rnd, staging and production you can set
 ``TEST_TIERS_ALLOWED=dev,rnd,staging,production``. Then you would be able to run tests marked ``tier_dev => true``, ``tier_production => true`` with ``TEST_TIERS=dev,production``
 
 Note, if the ``TEST_TIERS`` environment variable is set to empty string or nil, all tiers will be executed.

Rakefile

@@ -7,10 +7,25 @@ def gem_present(name)
   !Bundler.rubygems.find_name(name).empty?
 end
 
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb'].exclude('spec/fixtures/**/*_spec.rb')
+desc 'Runs unit tests'
+RSpec::Core::RakeTask.new(:'spec:unit') do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+                 .exclude('spec/fixtures/**/*_spec.rb')
+                 .exclude('spec/acceptance/**/*_spec.rb')
 end
 
+desc 'Runs acceptance tests'
+RSpec::Core::RakeTask.new(:'spec:acceptance') do |spec|
+  spec.pattern = FileList['spec/acceptance/**/*_spec.rb']
+end
+
+Rake::Task[:spec].clear
+desc 'Runs all tests'
+task spec: [
+  :'spec:unit',
+  :'spec:acceptance',
+]
+
 require 'yard'
 YARD::Rake::YardocTask.new
 

lib/puppetlabs_spec_helper/puppetlabs_spec/metadata.rb

@@ -0,0 +1,95 @@
+# Main module
+module PuppetlabsSpec; end
+# A Metadata JSON releated functions
+module PuppetlabsSpec::Metadata
+  # This method returns an array of dependencies from the metadata.json file
+  # in the format of an array of hashes, containing 'remote' (module_name) and
+  # optionally 'ref' (version) elements. If no dependencies are specified,
+  # empty array is returned
+  def module_dependencies_from_metadata(metadata_opts)
+    metadata = module_metadata
+    return [] unless metadata.key?('dependencies')
+
+    opts = metadata_opts['opts']
+    forge = if !opts.nil? && !opts['forge'].nil?
+              opts['forge']
+            else
+              'https://forge.puppet.com/'
+            end
+    dependencies = []
+    metadata['dependencies'].each do |dep|
+      tmp = { 'remote' => dep['name'].sub('/', '-') }
+
+      if dep.key?('version_requirement')
+        tmp['ref'] = module_version_from_requirement(
+          tmp['remote'], dep['version_requirement'], forge
+        )
+      end
+      dependencies.push(tmp)
+    end
+
+    dependencies
+  end
+
+  # This method uses the module_source_directory path to read the metadata.json
+  # file into a json array
+  def module_metadata
+    metadata_path = "#{module_source_dir}/metadata.json"
+    unless File.exist?(metadata_path)
+      raise "Error loading metadata.json file from #{module_source_dir}"
+    end
+    JSON.parse(File.read(metadata_path))
+  end
+
+  private
+
+  # This method takes a module name and the version requirement string from the
+  # metadata.json file, containing either lower bounds of version or both lower
+  # and upper bounds. The function then uses the forge rest endpoint to find
+  # the most recent release of the given module matching the version requirement
+  def module_version_from_requirement(mod_name, vr_str, forge_api)
+    require 'net/http'
+    forge_api = File.join(forge_api, '')
+    uri = URI.parse("#{forge_api}v3/modules/#{mod_name}")
+    req = Net::HTTP::Get.new(uri.request_uri)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = (uri.scheme == 'https')
+    req.basic_auth uri.user, uri.password unless uri.user.nil? || uri.password.nil?
+    response = http.request(req)
+    forge_data = JSON.parse(response.body)
+
+    vrs = version_requirements_from_string(vr_str)
+
+    # Here we iterate the releases of the given module and pick the most recent
+    # that matches to version requirement
+    forge_data['releases'].each do |rel|
+      return rel['version'] if vrs.all? { |vr| vr.match?('', rel['version']) }
+    end
+
+    raise "No release version found matching '#{vr_str}'"
+  end
+
+  # This method takes a version requirement string as specified in the link
+  # below, with either simply a lower bound, or both lower and upper bounds and
+  # returns an array of Gem::Dependency objects
+  # https://docs.puppet.com/puppet/latest/modules_metadata.html
+  def version_requirements_from_string(vr_str)
+    ops = vr_str.scan(%r{[(<|>|=)]{1,2}}i)
+    vers = vr_str.scan(%r{[(0-9|\.)]+}i)
+
+    raise 'Invalid version requirements' if ops.count != 0 &&
+                                            ops.count != vers.count
+
+    vrs = []
+    ops.each_with_index do |op, index|
+      vrs.push(Gem::Dependency.new('', "#{op} #{vers[index]}"))
+    end
+
+    vrs
+  end
+
+  # This is a helper for the self-symlink entry of fixtures.yml
+  def module_source_dir
+    Dir.pwd
+  end
+end