Merge pull request #106 from aiperon/introduce_s3_manager

Introduce Amazon S3 StorageManager

app/models/manageiq/providers/amazon/cloud_manager.rb

@@ -23,6 +23,12 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
 
   include ManageIQ::Providers::Amazon::ManagerMixin
 
+  has_many :storage_managers,
+           :foreign_key => :parent_ems_id,
+           :class_name  => "ManageIQ::Providers::StorageManager",
+           :autosave    => true,
+           :dependent   => :destroy
+
   has_one :network_manager,
           :foreign_key => :parent_ems_id,
           :class_name  => "ManageIQ::Providers::Amazon::NetworkManager",
@@ -58,6 +64,17 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
            :to        => :ebs_storage_manager,
            :allow_nil => true
 
+  has_one :s3_storage_manager,
+          :foreign_key => :parent_ems_id,
+          :class_name  => "ManageIQ::Providers::Amazon::StorageManager::S3",
+          :autosave    => true,
+          :dependent   => :destroy
+
+  delegate :cloud_object_store_containers,
+           :cloud_object_store_objects,
+           :to        => :s3_storage_manager,
+           :allow_nil => true
+
   before_create :ensure_managers
 
   supports :provisioning
@@ -74,6 +91,11 @@ def ensure_managers
     ebs_storage_manager.name            = "#{name} EBS Storage Manager"
     ebs_storage_manager.zone_id         = zone_id
     ebs_storage_manager.provider_region = provider_region
+
+    build_s3_storage_manager unless s3_storage_manager
+    s3_storage_manager.name            = "#{name} S3 Storage Manager"
+    s3_storage_manager.zone_id         = zone_id
+    s3_storage_manager.provider_region = provider_region
   end
 
   def self.ems_type

app/models/manageiq/providers/amazon/inventory.rb

@@ -28,6 +28,10 @@ def aws_elb
     @aws_elb ||= ems.connect(:service => :ElasticLoadBalancing)
   end
 
+  def aws_s3
+    @aws_s3 ||= ems.connect(:service => :S3)
+  end
+
   def instances
     []
   end

app/models/manageiq/providers/amazon/inventory/factory.rb

@@ -16,6 +16,8 @@ def target(ems, target)
         ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager.new(ems, target)
       when ManageIQ::Providers::Amazon::StorageManager::Ebs
         ManageIQ::Providers::Amazon::Inventory::Targets::StorageManager::Ebs.new(ems, target)
+      when ManageIQ::Providers::Amazon::StorageManager::S3
+        ManageIQ::Providers::Amazon::Inventory::Targets::StorageManager::S3.new(ems, target)
       else
         ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target)
       end

app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb

@@ -294,4 +294,20 @@ def cloud_volume_snapshots_init_data(extra_attributes = {})
 
     init_data(::ManageIQ::Providers::Amazon::StorageManager::Ebs::CloudVolumeSnapshot, attributes, extra_attributes)
   end
+
+  def cloud_object_store_containers_init_data(extra_attributes = {})
+    attributes = {
+      :association => :cloud_object_store_containers,
+    }
+
+    init_data(::CloudObjectStoreContainer, attributes, extra_attributes)
+  end
+
+  def cloud_object_store_objects_init_data(extra_attributes = {})
+    attributes = {
+      :association => :cloud_object_store_objects,
+    }
+
+    init_data(::CloudObjectStoreObject, attributes, extra_attributes)
+  end
 end

app/models/manageiq/providers/amazon/inventory/targets/storage_manager/s3.rb

@@ -0,0 +1,14 @@
+class ManageIQ::Providers::Amazon::Inventory::Targets::StorageManager::S3 <
+  ManageIQ::Providers::Amazon::Inventory::Targets
+  def initialize_inventory_collections
+    add_inventory_collections(%i(cloud_object_store_containers cloud_object_store_objects))
+  end
+
+  def cloud_object_store_containers
+    HashCollection.new(aws_s3.client.list_buckets.buckets)
+  end
+
+  def cloud_object_store_objects
+    HashCollection.new([])
+  end
+end

app/models/manageiq/providers/amazon/storage_manager/s3.rb

@@ -0,0 +1,36 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3 < ManageIQ::Providers::StorageManager
+  require_nested :RefreshParser
+  require_nested :RefreshWorker
+  require_nested :Refresher
+
+  include ManageIQ::Providers::Amazon::ManagerMixin
+  include ManageIQ::Providers::StorageManager::ObjectMixin
+
+  delegate :authentication_check,
+           :authentication_status,
+           :authentications,
+           :authentication_for_summary,
+           :zone,
+           :connect,
+           :verify_credentials,
+           :with_provider_connection,
+           :address,
+           :ip_address,
+           :hostname,
+           :default_endpoint,
+           :endpoints,
+           :to        => :parent_manager,
+           :allow_nil => true
+
+  def self.ems_type
+    @ems_type ||= "s3".freeze
+  end
+
+  def self.description
+    @description ||= "Amazon S3".freeze
+  end
+
+  def self.hostname_required?
+    false
+  end
+end

app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser.rb

@@ -0,0 +1,52 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParser
+  include ManageIQ::Providers::Amazon::RefreshHelperMethods
+
+  def initialize(ems, options = nil)
+    @ems        = ems
+    @aws_s3     = ems.connect(:service => :S3)
+    @data       = {}
+    @data_index = {}
+    @options    = options || {}
+  end
+
+  def ems_inv_to_hashes
+    log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]"
+
+    $aws_log.info("#{log_header}...")
+    object_store
+
+    $aws_log.info("#{log_header}...Complete")
+
+    @data
+  end
+
+  def object_store
+    buckets = @aws_s3.client.list_buckets.buckets
+    process_collection(buckets, :cloud_object_store_containers) { |c| parse_container(c) }
+  end
+
+  def parse_container(bucket)
+    uid = bucket.name
+
+    new_result = {
+      :ems_ref => uid,
+      :key     => bucket.name
+    }
+    return uid, new_result
+  end
+
+  def parse_object(obj, bucket)
+    uid = obj.key
+
+    new_result = {
+      :ems_ref        => uid,
+      :etag           => obj.etag,
+      :last_modified  => obj.last_modified,
+      :content_length => obj.size,
+      :key            => obj.key,
+      #:content_type   => obj.content_type,
+      :container      => bucket
+    }
+    return uid, new_result
+  end
+end

app/models/manageiq/providers/amazon/storage_manager/s3/refresh_parser_inventory_object.rb

@@ -0,0 +1,30 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParserInventoryObject < ::ManagerRefresh::RefreshParserInventoryObject
+  include ManageIQ::Providers::Amazon::RefreshHelperMethods
+
+  def populate_inventory_collections
+    log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{inventory.ems.name}] id: [#{inventory.ems.id}]"
+
+    $aws_log.info("#{log_header}...}")
+    object_store
+    $aws_log.info("#{log_header}...Complete")
+
+    inventory_collections
+  end
+
+  private
+
+  def object_store
+    process_inventory_collection(
+      inventory.cloud_object_store_containers,
+      :cloud_object_store_containers
+    ) { |c| parse_container(c) }
+  end
+
+  def parse_container(bucket)
+    uid = bucket['name']
+    {
+      :ems_ref => uid,
+      :key     => bucket['name']
+    }
+  end
+end

app/models/manageiq/providers/amazon/storage_manager/s3/refresh_worker.rb

@@ -0,0 +1,11 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3::RefreshWorker < ::MiqEmsRefreshWorker
+  require_nested :Runner
+
+  def self.ems_class
+    ManageIQ::Providers::Amazon::StorageManager::S3
+  end
+
+  def self.settings_name
+    :ems_refresh_worker_amazon_s3
+  end
+end

app/models/manageiq/providers/amazon/storage_manager/s3/refresh_worker/runner.rb

@@ -0,0 +1,3 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3::RefreshWorker::Runner <
+  ManageIQ::Providers::BaseManager::RefreshWorker::Runner
+end

app/models/manageiq/providers/amazon/storage_manager/s3/refresher.rb

@@ -0,0 +1,39 @@
+class ManageIQ::Providers::Amazon::StorageManager::S3::Refresher < ManageIQ::Providers::BaseManager::Refresher
+  include ::EmsRefresh::Refreshers::EmsRefresherMixin
+
+  def collect_inventory_for_targets(ems, targets)
+    targets_with_data = targets.collect do |target|
+      target_name = target.try(:name) || target.try(:event_type)
+
+      _log.info "Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]..."
+
+      if refresher_options.try(:[], :inventory_object_refresh)
+        inventory = ManageIQ::Providers::Amazon::Inventory::Factory.inventory(ems, target)
+      end
+
+      _log.info "Filtering inventory...Complete"
+      [target, inventory]
+    end
+
+    targets_with_data
+  end
+
+  def parse_targeted_inventory(ems, _target, inventory)
+    log_header = format_ems_for_logging(ems)
+    _log.debug "#{log_header} Parsing inventory..."
+    hashes, = Benchmark.realtime_block(:parse_inventory) do
+      if refresher_options.try(:[], :inventory_object_refresh)
+        ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParserInventoryObject.new(inventory).populate_inventory_collections
+      else
+        ManageIQ::Providers::Amazon::StorageManager::S3::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+      end
+    end
+    _log.debug "#{log_header} Parsing inventory...Complete"
+
+    hashes
+  end
+
+  def post_process_refresh_classes
+    []
+  end
+end

config/settings.yml

@@ -44,3 +44,4 @@
         :ems_refresh_worker_amazon: {}
         :ems_refresh_worker_amazon_network: {}
         :ems_refresh_worker_amazon_ebs_storage: {}
+        :ems_refresh_worker_amazon_s3: {}

spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb

@@ -33,7 +33,7 @@ def expected_table_counts
       :cloud_subnet                  => 10,
       :custom_attribute              => 0,
       :disk                          => 10,
-      :ext_management_system         => 3,
+      :ext_management_system         => 4,
       :firewall_rule                 => 119,
       :flavor                        => 56,
       :floating_ip                   => 12,

spec/models/manageiq/providers/amazon/aws_stubs.rb

@@ -32,6 +32,7 @@ def test_counts(scaling = nil)
       :outbound_firewall_rule_per_security_group_count => scaling * 5,
       :cloud_volume_count                              => scaling * 5,
       :cloud_volume_snapshot_count                     => scaling * 5,
+      :s3_buckets_count                                => scaling * 5
     }
   end
 
@@ -346,6 +347,17 @@ def mocked_load_balancers
     mocked_lbs
   end
 
+  def mocked_s3_buckets
+    mocked_s3_buckets = []
+    test_counts[:s3_buckets_count].times do |i|
+      mocked_s3_buckets << {
+        :name          => "bucket_id_#{i}",
+        :creation_date => Time.now.utc
+      }
+    end
+    mocked_s3_buckets
+  end
+
   def mocked_instance_health
     mocked_instance_healths = []
     expected_table_counts[:load_balancer_pool_member].times do |i|

spec/models/manageiq/providers/amazon/cloud_manager/refresher_other_region_spec.rb

@@ -34,7 +34,7 @@
   end
 
   def assert_table_counts
-    expect(ExtManagementSystem.count).to eq(3)
+    expect(ExtManagementSystem.count).to eq(4)
     expect(Flavor.count).to eq(56)
     expect(AvailabilityZone.count).to eq(3)
     expect(FloatingIp.count).to eq(3)

spec/models/manageiq/providers/amazon/cloud_manager/stubbed_refresher_spec.rb

@@ -132,7 +132,7 @@ def expected_table_counts(disconnect = nil)
 
     {
       :auth_private_key                  => test_counts[:key_pair_count],
-      :ext_management_system             => 3,
+      :ext_management_system             => 4,
       # TODO(lsmola) collect all flavors for original refresh
       :flavor                            => @inventory_object_settings[:inventory_object_refresh] ? 57 : 56,
       :availability_zone                 => 5,

spec/models/manageiq/providers/amazon/network_manager/stubbed_refresher_spec.rb

@@ -111,7 +111,7 @@ def expected_table_counts
 
     {
       :auth_private_key                  => 0,
-      :ext_management_system             => 3,
+      :ext_management_system             => 4,
       :flavor                            => 0,
       :availability_zone                 => 0,
       :vm_or_template                    => 0,

spec/models/manageiq/providers/amazon/storage_manager/ebs/stubbed_refresher_spec.rb

@@ -86,7 +86,7 @@ def stub_responses
   def expected_table_counts
     {
       :auth_private_key                  => 0,
-      :ext_management_system             => 3,
+      :ext_management_system             => 4,
       :flavor                            => 0,
       :availability_zone                 => 0,
       :vm_or_template                    => 0,