Additional fixes

Author: jakecoll

blazar/enforcement/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Bull.
+# Copyright (c) 2020 University of Chicago.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

blazar/enforcement/enforcement.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Bull.
+# Copyright (c) 2020 University of Chicago.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from blazar.enforcement import filters
+from blazar.utils.openstack import base
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -23,9 +24,7 @@
 enforcement_opts = [
     cfg.ListOpt('enabled_filters',
                 default=[],
-                help='Maximum lease duration in seconds. If this is set to -1,'
-                     ' there is not limit. For active leases being updated, '
-                     'the limit applies between now and the new end date.'),
+                help='List of enabled usage enforcement filters.'),
 ]
 
 CONF.register_opts(enforcement_opts, group='enforcement')
@@ -46,12 +45,9 @@ def load_filters(self):
     def format_context(self, context):
         ctx = context.to_dict()
         region_name = CONF.os_region_name
-        identity_endpoints = [
-            x['endpoints'] for x in ctx['service_catalog']
-            if x['type'] == 'identity'][0]
-        auth_url = [
-            x['url'] for x in identity_endpoints
-            if (x['interface'] == 'public' and x['region'] == region_name)][0]
+        auth_url = base.url_for(
+            ctx['service_catalog'], CONF.identity_service,
+            os_region_name=region_name)
 
         return dict(user_id=ctx['user_id'], project_id=ctx['project_id'],
                     auth_url=auth_url, region_name=region_name)
@@ -64,7 +60,7 @@ def format_lease(self, lease_values, reservations, allocations):
             res = reservation.copy()
             resource_type = res['resource_type']
             res['allocations'] = allocations[resource_type]
-            lease['reservations'] = reservations
+            lease['reservations'].append(reservations)
 
         return lease
 

blazar/enforcement/filters/base_filter.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Bull.
+# Copyright (c) 2020 University of Chicago.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

blazar/enforcement/filters/external_service_filter.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Bull.
+# Copyright (c) 2020 University of Chicago.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,9 +15,9 @@
 
 import requests
 
+from blazar.enforcement.filters import base_filter
 from blazar import exceptions
 from blazar.i18n import _
-from blazar.enforcement.filters import base_filter
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -30,6 +30,7 @@ class ExternalServiceUnsupportedHTTPResponse(exceptions.BlazarExceptions):
     msg_fmt = _('External Service Filter returned a %(status)s http response. '
                 'Only 204 and 403 responses are supported.')
 
+
 class ExternalServiceFilterException(exceptions.BlazarExceptions):
     code = 400
     msg_fmt = _('%(message)s')
@@ -84,7 +85,6 @@ def check_create(self, context, lease_values):
 
             self.post(path, body)
 
-
     def check_update(self, context, current_lease_values, new_lease_values):
         if self.external_service_endpoint:
             path = '/v1/check-update'

blazar/enforcement/filters/max_reservation_length_filter.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2013 Bull.
+# Copyright (c) 2020 University of Chicago.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timedelta
+from datetime import datetime
+from datetime import timedelta
 
+from blazar.enforcement.filters import base_filter
 from blazar import exceptions
 from blazar.i18n import _
-from blazar.enforcement.filters import base_filter
 
 from oslo_config import cfg
 from oslo_log import log as logging

blazar/manager/service.py

@@ -219,6 +219,44 @@ def _date_from_string(self, date_string, date_format=LEASE_DATE_FORMAT):
                                          date_format=date_format)
         return date
 
+    def _parse_lease_dates(self, start_date, end_date):
+        now = datetime.datetime.utcnow()
+        now = datetime.datetime(now.year,
+                                now.month,
+                                now.day,
+                                now.hour,
+                                now.minute)
+        if start_date == 'now':
+            start_date = now
+        else:
+            start_date = self._date_from_string(start_date)
+        if end_date == 'now':
+            end_date = now
+        else:
+            end_date = self._date_from_string(end_date)
+
+        return start_date, end_date, now
+
+    def _check_for_invalid_date_inputs(self, lease, values, now):
+        if (lease['start_date'] < now and
+                values['start_date'] != lease['start_date']):
+            raise common_ex.InvalidInput(
+                'Cannot modify the start date of already started leases')
+
+        if (lease['start_date'] > now and
+                values['start_date'] < now):
+            raise common_ex.InvalidInput(
+                'Start date must be later than current date')
+
+        if lease['end_date'] < now:
+            raise common_ex.InvalidInput(
+                'Terminated leases can only be renamed')
+
+        if (values['end_date'] < now or
+           values['end_date'] < values['start_date']):
+            raise common_ex.InvalidInput(
+                'End date must be later than current and start date')
+
     def validate_params(self, values, required_params):
         if isinstance(required_params, list):
             required_params = set(required_params)
@@ -253,20 +291,8 @@ def create_lease(self, lease_values):
             self.validate_params(res, ['resource_type'])
 
         # Create the lease without the reservations
-        start_date = lease_values['start_date']
-        end_date = lease_values['end_date']
-
-        now = datetime.datetime.utcnow()
-        now = datetime.datetime(now.year,
-                                now.month,
-                                now.day,
-                                now.hour,
-                                now.minute)
-        if start_date == 'now':
-            start_date = now
-        else:
-            start_date = self._date_from_string(start_date)
-        end_date = self._date_from_string(end_date)
+        start_date, end_date, now = self._parse_lease_dates(
+            lease_values['start_date'], lease_values['end_date'])
 
         if start_date < now:
             raise common_ex.InvalidInput(
@@ -384,42 +410,13 @@ def update_lease(self, lease_id, values):
             datetime.datetime.strftime(lease['end_date'], LEASE_DATE_FORMAT))
         before_end_date = values.get('before_end_date', None)
 
-        now = datetime.datetime.utcnow()
-        now = datetime.datetime(now.year,
-                                now.month,
-                                now.day,
-                                now.hour,
-                                now.minute)
-        if start_date == 'now':
-            start_date = now
-        else:
-            start_date = self._date_from_string(start_date)
-        if end_date == 'now':
-            end_date = now
-        else:
-            end_date = self._date_from_string(end_date)
+        start_date, end_date, now = self._parse_lease_dates(start_date,
+                                                            end_date)
 
         values['start_date'] = start_date
         values['end_date'] = end_date
 
-        if (lease['start_date'] < now and
-                values['start_date'] != lease['start_date']):
-            raise common_ex.InvalidInput(
-                'Cannot modify the start date of already started leases')
-
-        if (lease['start_date'] > now and
-                values['start_date'] < now):
-            raise common_ex.InvalidInput(
-                'Start date must be later than current date')
-
-        if lease['end_date'] < now:
-            raise common_ex.InvalidInput(
-                'Terminated leases can only be renamed')
-
-        if (values['end_date'] < now or
-           values['end_date'] < values['start_date']):
-            raise common_ex.InvalidInput(
-                'End date must be later than current and start date')
+        self._check_for_invalid_date_inputs(lease, values, now)
 
         with trusts.create_ctx_from_trust(lease['trust_id']) as ctx:
             if before_end_date:
@@ -478,6 +475,9 @@ def update_lease(self, lease_id, values):
                 resource_type = v.get('resource_type',
                                       reservation['resource_type'])
 
+                if resource_type != reservation['resource_type']:
+                    raise exceptions.CantUpdateParameter(
+                        param='resource_type')
                 self.plugins[resource_type].update_reservation(
                     reservation['id'], v)
 
@@ -584,7 +584,7 @@ def end_lease(self, lease_id, event_id):
         lease = self.get_lease(lease_id)
         allocations = self._existing_allocations(lease['reservations'])
 
-        with trusts.create_ctx_from_trust(lease['trust_id']) as ctx:
+        with trusts.create_ctx_from_trust(lease['trust_id']):
             self.enforcement.on_end(context.current(), lease, allocations)
             self._basic_action(lease_id, event_id, 'on_end',
                                status.reservation.DELETED)

blazar/plugins/instances/instance_plugin.py

@@ -44,6 +44,7 @@
 INSTANCE_DELETION_TIMEOUT = 10 * 60 * 1000  # 10 minutes
 
 NONE_VALUES = ('None', 'none', None)
+QUERY_TYPE_ALLOCATION = 'allocation'
 
 
 class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper):
@@ -138,6 +139,43 @@ def get_hosts_list(self, host_info, cpus, memory, disk):
             used_disk += disk
         return hosts_list
 
+    def allocation_candidates(self, reservation):
+        return self.pickup_hosts(self, None, reservation)['added']
+
+    def list_allocations(self, query):
+        hosts_id_list = [h['id'] for h in db_api.host_list()]
+        options = self.get_query_options(query, QUERY_TYPE_ALLOCATION)
+
+        hosts_allocations = self.query_allocations(hosts_id_list, **options)
+        return [{"resource_id": host, "reservations": allocs}
+                for host, allocs in hosts_allocations.items()]
+
+    def query_allocations(self, hosts, lease_id=None, reservation_id=None):
+        """Return dict of host and its allocations.
+
+        The list element forms
+        {
+          'host-id': [
+                       {
+                         'lease_id': lease_id,
+                         'id': reservation_id
+                       },
+                     ]
+        }.
+        """
+        start = datetime.datetime.utcnow()
+        end = datetime.date.max
+
+        # To reduce overhead, this method only executes one query
+        # to get the allocation information
+        rsv_lease_host = db_utils.get_reservation_allocations_by_host_ids(
+            hosts, start, end, lease_id, reservation_id)
+
+        hosts_allocs = collections.defaultdict(list)
+        for rsv, lease, host in rsv_lease_host:
+            hosts_allocs[host].append({'lease_id': lease, 'id': rsv})
+        return hosts_allocs
+
     def query_available_hosts(self, cpus=None, memory=None, disk=None,
                               resource_properties=None,
                               start_date=None, end_date=None,
@@ -759,12 +797,12 @@ def _post_reallocate(self, reservation, lease, host_id, num):
         LOG.warn('Resource changed for reservation %s (lease: %s).',
                  reservation['id'], lease['name'])
 
-    def get(self, resource_id):
-        return None
-
-    def query_allocations(self, resource_id_list, lease_id=None,
-                          reservation_id=None):
-        return None
-
-    def allocation_candidates(self, lease_values):
-        return None
+    def get(self, host_id):
+        host = db_api.host_get(host_id)
+        extra_capabilities = self._get_extra_capabilities(host_id)
+        if host is not None and extra_capabilities:
+            res = host.copy()
+            res.update(extra_capabilities)
+            return res
+        else:
+            return host