Fix acceptance tests

.sync.yml

@@ -1,3 +1,7 @@
 ---
+Gemfile:
+  optional:
+    ':system_tests':
+      - gem: rspec-retry
 spec/spec_helper_acceptance.rb:
   unmanaged: false

Gemfile

@@ -17,6 +17,7 @@ end
 
 group :system_tests do
   gem 'voxpupuli-acceptance', '~> 1.0',  :require => false
+  gem 'rspec-retry',                     :require => false
 end
 
 group :release do

spec/helpers/acceptance/tests/basic_shared_examples.rb

@@ -22,15 +22,17 @@
 
     describe port(5602) { it { is_expected.to be_listening } }
 
-    describe server :container do
-      describe http("http://localhost:5602#{request_path}") do
-        # Kibana versions reply to requests differently depending upon whether
-        # Elasticsearch is up and responding on the backend. In most cases we
-        # just want to ensure that Kibana has installed and started, so testing
-        # to confirm whether Kibana is replying with proper HTTP response codes
-        # is sufficient (earlier versions return 200 in most cases, later
-        # versions pass through ES unavailability as 503's).
-        it('returns OK', :api) { expect(response.status).to eq(200).or(eq(503)) }
+    describe "http://localhost:5602#{request_path}" do
+      # Kibana versions reply to requests differently depending upon whether
+      # Elasticsearch is up and responding on the backend. In most cases we
+      # just want to ensure that Kibana has installed and started, so testing
+      # to confirm whether Kibana is replying with proper HTTP response codes
+      # is sufficient (earlier versions return 200 in most cases, later
+      # versions pass through ES unavailability as 503's).
+      subject { shell("curl -o /dev/null -w '%{http_code}' http://localhost:5602#{request_path}") }
+
+      it('returns OK', :api) do
+        expect(subject.stdout.to_i).to eq(200).or(eq(503))
       end
     end
   end

spec/support/acceptance/kibana.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'rspec/retry'
+
+require_relative '../../spec_utilities'
+
+ENV['PUPPET_INSTALL_TYPE'] = 'agent' if ENV['PUPPET_INSTALL_TYPE'].nil?
+
+RSpec.configure do |c|
+  # Project root
+  # proj_root = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+  # Readable test descriptions
+  c.formatter = :documentation
+
+  c.add_setting :pkg_ext
+  c.pkg_ext = case fact('osfamily')
+              when 'Debian'
+                'deb'
+              when 'RedHat'
+                'rpm'
+              end
+
+  c.add_setting :is_snapshot
+  c.is_snapshot = c.files_to_run.any? { |fn| fn.include? 'snapshot' }
+
+  c.add_setting :oss
+
+  # Copy over the snapshot package if we're running snapshot tests
+  if c.is_snapshot && !c.pkg_ext.nil?
+    c.add_setting :snapshot_file
+    c.snapshot_file = "kibana-snapshot.#{c.pkg_ext}"
+
+    c.add_setting :snapshot_version
+    c.snapshot_version = File.readlink(artifact(c.snapshot_file)).match(%r{kibana(?:-oss)?-(?<v>.*)[.][a-z]+})[:v]
+
+    c.oss = (!File.readlink(artifact(c.snapshot_file)).match(%r{-oss}).nil?)
+  else
+    c.oss = false
+  end
+
+  # Configure all nodes in nodeset
+  c.before :suite do
+    if c.is_snapshot
+      hosts.each do |host|
+        scp_to host, artifact(c.snapshot_file), "/tmp/#{c.snapshot_file}"
+      end
+    end
+  end
+
+  c.around :each, :api do |example|
+    # The initial optimization startup time of Kibana is _incredibly_ slow,
+    # so we need to be pretty generous with how we retry API call attempts.
+    example.run_with_retry retry: 10, retry_wait: 5
+  end
+end