From a21f88e28be56f0fbe2df5de252fb51f4c06cb6a Mon Sep 17 00:00:00 2001 From: Tetsuro Nakamura Date: Fri, 21 Dec 2018 01:38:51 +0000 Subject: [PATCH] Change pickup_hosts() for affinity=True/None The pickup_hosts() function, called for create/update/reallocate instance reservation, assumed affinity=False and returned sets of host ids to add/remove. To support affinity=True/None, this patch changes pickup_hosts() to return lists of hosts ids to add/remove allowing duplicate host ids according to the number of instances to reserve on that host. Blueprint: no-affinity-instance-reservation Change-Id: If1f4d7c3aee91dc32a173bd4ae598d194f38b3ea --- blazar/plugins/instances/instance_plugin.py | 175 ++++++++----- .../plugins/instances/test_instance_plugin.py | 234 ++++++++++++++++-- blazar/tests/utils/test_plugins.py | 22 ++ blazar/utils/plugins.py | 25 ++ 4 files changed, 374 insertions(+), 82 deletions(-) diff --git a/blazar/plugins/instances/instance_plugin.py b/blazar/plugins/instances/instance_plugin.py index f5190f06..d35cade0 100644 --- a/blazar/plugins/instances/instance_plugin.py +++ b/blazar/plugins/instances/instance_plugin.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import datetime import retrying @@ -60,6 +61,15 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): self.monitor.register_healing_handler(self.heal_reservations) self.placement_client = placement.BlazarPlacementClient() + # TODO(tetsuro) Remove this with a release note when all the support + # for True/None affinity is ready + def _check_affinity(self, affinity): + # TODO(masahito) the instance reservation plugin only supports + # anti-affinity rule in short-term goal. + if bool_from_string(affinity): + raise mgr_exceptions.MalformedParameter( + param='affinity (only affinity = False is supported)') + def filter_hosts_by_reservation(self, hosts, start_date, end_date, excludes): free = [] @@ -75,7 +85,7 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): if r['id'] not in excludes] if reservations == []: - free.append({'host': host, 'reservations': None}) + free.append({'host': host, 'reservations': []}) elif not [r for r in reservations if r['resource_type'] == oshosts.RESOURCE_TYPE]: non_free.append({'host': host, 'reservations': reservations}) @@ -117,13 +127,37 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): return max_vcpus, max_memory, max_disk + def get_hosts_list(self, host_info, cpus, memory, disk): + hosts_list = [] + host = host_info['host'] + reservations = host_info['reservations'] + max_cpus, max_memory, max_disk = self.max_usages(host, + reservations) + used_cpus, used_memory, used_disk = (cpus, memory, disk) + while (max_cpus + used_cpus <= host['vcpus'] and + max_memory + used_memory <= host['memory_mb'] and + max_disk + used_disk <= host['local_gb']): + hosts_list.append(host) + used_cpus += cpus + used_memory += memory + used_disk += disk + return hosts_list + def query_available_hosts(self, cpus=None, memory=None, disk=None, resource_properties=None, start_date=None, end_date=None, excludes_res=None): - """Query hosts that are available for a reservation. + """Returns a list of available hosts for a reservation. - Its return value is in the order of reserved hosts to free hosts now. + The list is in the order of reserved hosts to free hosts. + + 1. filter hosts that have a spec enough to accommodate the flavor + 2. categorize hosts into hosts with and without allocation + at the reservation time frame + 3. filter out hosts used by physical host reservation from + allocate_host + 4. filter out hosts that can't accommodate the flavor at the + time frame because of other reservations """ flavor_definitions = [ 'and', @@ -145,34 +179,28 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): excludes_res) available_hosts = [] - for host_info in reserved_hosts: - host = host_info['host'] - reservations = host_info['reservations'] - max_cpus, max_memory, max_disk = self.max_usages(host, - reservations) + for host_info in (reserved_hosts + free_hosts): + hosts_list = self.get_hosts_list(host_info, cpus, memory, disk) + available_hosts.extend(hosts_list) - if not (max_cpus + cpus > host['vcpus'] or - max_memory + memory > host['memory_mb'] or - max_disk + disk > host['local_gb']): - available_hosts.append(host) - - available_hosts.extend([h['host'] for h in free_hosts]) return available_hosts def pickup_hosts(self, reservation_id, values): - """Checks whether Blazar can accommodate the request. + """Returns lists of host ids to add/remove. - This method filters and pick up hosts for this reservation - with following steps. + This function picks up available hosts, calculates the difference from + old reservations and returns a dict of a list of host ids to add + and remove keyed by "added" or "removed". - 1. filter hosts that have a spec enough to accommodate the flavor - 2. categorize hosts allocated_hosts and not_allocated_hosts - at the reservation time frame - 3. filter out hosts used by physical host reservation from - allocate_host - 4. filter out hosts that can't accommodate the flavor at the - time frame because of others reservations + Note that the lists allow duplicated host ids for `affinity=True` + cases. + + :raises: NotEnoughHostsAvailable exception if there are not enough + hosts available for the request """ + req_amount = values['amount'] + affinity = bool_from_string(values['affinity'], default=None) + query_params = { 'cpus': values['vcpus'], 'memory': values['memory_mb'], @@ -182,32 +210,69 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): 'end_date': values['end_date'] } - # add the specific query param for reservation update old_allocs = db_api.host_allocation_get_all_by_values( reservation_id=reservation_id) if old_allocs: + # This is a path for *update* reservation. Add the specific + # query param not to consider resources reserved by existing + # reservations to update query_params['excludes_res'] = [reservation_id] new_hosts = self.query_available_hosts(**query_params) - old_host_ids = {h['compute_host_id'] for h in old_allocs} - candidate_ids = {h['id'] for h in new_hosts} + old_host_id_list = [h['compute_host_id'] for h in old_allocs] + candidate_id_list = [h['id'] for h in new_hosts] - kept_host_ids = old_host_ids & candidate_ids - removed_host_ids = old_host_ids - candidate_ids - extra_host_ids = candidate_ids - old_host_ids - added_host_ids = set([]) + # Build `new_host_id_list`. Note that we'd like to pick up hosts in + # the following order of priority: + # 1. hosts reserved by the reservation to update + # 2. hosts with reservations followed by hosts without reservations + # Note that the `candidate_id_list` has already been ordered + # satisfying the second requirement. + if affinity: + host_id_map = collections.Counter(candidate_id_list) + available = {k for k, v in host_id_map.items() if v >= req_amount} + if not available: + raise mgr_exceptions.NotEnoughHostsAvailable() + new_host_ids = set(old_host_id_list) & available + if new_host_ids: + # (priority 1) This is a path for update reservation. We pick + # up a host from hosts reserved by the reservation to update. + new_host_id = new_host_ids.pop() + else: + # (priority 2) This is a path both for update and for new + # reservation. We pick up hosts with some other reservations + # if possible and otherwise pick up hosts without any + # reservation. We can do so by considering the order of the + # `candidate_id_list`. + for host_id in candidate_id_list: + if host_id in available: + new_host_id = host_id + break + new_host_id_list = [new_host_id] * req_amount + else: + # Hosts that can accommodate but don't satisfy priority 1 + _, possible_host_list = plugins_utils.list_difference( + old_host_id_list, candidate_id_list) + # Hosts that satisfy priority 1 + new_host_id_list, _ = plugins_utils.list_difference( + candidate_id_list, possible_host_list) + if affinity is False: + # Eliminate the duplication + new_host_id_list = list(set(new_host_id_list)) + for host_id in possible_host_list: + if (affinity is False) and (host_id in new_host_id_list): + # Eliminate the duplication + continue + new_host_id_list.append(host_id) + if len(new_host_id_list) < req_amount: + raise mgr_exceptions.NotEnoughHostsAvailable() + while len(new_host_id_list) > req_amount: + new_host_id_list.pop() - if len(kept_host_ids) > values['amount']: - extra = len(kept_host_ids) - values['amount'] - for i in range(extra): - removed_host_ids.add(kept_host_ids.pop()) - elif len(kept_host_ids) < values['amount']: - less = values['amount'] - len(kept_host_ids) - ordered_extra_host_ids = [h['id'] for h in new_hosts - if h['id'] in extra_host_ids] - for i in range(min(less, len(extra_host_ids))): - added_host_ids.add(ordered_extra_host_ids[i]) + # Calculate the difference from the existing reserved host + removed_host_ids, added_host_ids = plugins_utils.list_difference( + old_host_id_list, new_host_id_list) return {'added': added_host_ids, 'removed': removed_host_ids} @@ -328,26 +393,17 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): def reserve_resource(self, reservation_id, values): self.validate_reservation_param(values) - # TODO(masahito) the instance reservation plugin only supports - # anti-affinity rule in short-term goal. - if bool_from_string(values['affinity']): - raise mgr_exceptions.MalformedParameter( - param='affinity (only affinity = False is supported)') + self._check_affinity(values['affinity']) hosts = self.pickup_hosts(reservation_id, values) - if len(hosts['added']) < values['amount']: - raise mgr_exceptions.HostNotFound("The reservation can't be " - "accommodate because of less " - "capacity.") - instance_reservation_val = { 'reservation_id': reservation_id, 'vcpus': values['vcpus'], 'memory_mb': values['memory_mb'], 'disk_gb': values['disk_gb'], 'amount': values['amount'], - 'affinity': bool_from_string(values['affinity']), + 'affinity': bool_from_string(values['affinity'], default=None), 'resource_properties': values['resource_properties'] } instance_reservation = db_api.instance_reservation_create( @@ -396,11 +452,8 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): - If an instance reservation has already started - only amount is increasable. """ - # TODO(masahito) the instance reservation plugin only supports - # anti-affinity rule in short-term goal. - if bool_from_string(new_values.get('affinity', None)): - raise mgr_exceptions.MalformedParameter( - param='affinity (only affinity = False is supported)') + affinity = new_values.get('affinity', None) + self._check_affinity(affinity) reservation = db_api.reservation_get(reservation_id) lease = db_api.lease_get(reservation['lease_id']) @@ -422,6 +475,10 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): msg = "An error reservation doesn't accept an updating request." raise mgr_exceptions.InvalidStateUpdate(msg) + if new_values.get('affinity', None): + new_values['affinity'] = bool_from_string(new_values['affinity'], + default=None) + for key in updatable: if key not in new_values: new_values[key] = reservation[key] @@ -435,10 +492,6 @@ class VirtualInstancePlugin(base.BasePlugin, nova.NovaClientWrapper): "active status.") raise mgr_exceptions.CantUpdateParameter(err_msg) - if (new_values['amount'] - reservation['amount'] != - (len(changed_hosts['added']) - len(changed_hosts['removed']))): - raise mgr_exceptions.NotEnoughHostsAvailable() - db_api.instance_reservation_update( reservation['resource_id'], {key: new_values[key] for key in updatable}) diff --git a/blazar/tests/plugins/instances/test_instance_plugin.py b/blazar/tests/plugins/instances/test_instance_plugin.py index d252b993..4b4297f1 100644 --- a/blazar/tests/plugins/instances/test_instance_plugin.py +++ b/blazar/tests/plugins/instances/test_instance_plugin.py @@ -165,9 +165,9 @@ class TestVirtualInstancePlugin(tests.TestCase): 'get_reservations_by_host_id') mock_get_reservations.side_effect = fake_get_reservation_by_host - free = [{'host': hosts_list[0], 'reservations': None}, - {'host': hosts_list[1], 'reservations': None}, - {'host': hosts_list[2], 'reservations': None}] + free = [{'host': hosts_list[0], 'reservations': []}, + {'host': hosts_list[1], 'reservations': []}, + {'host': hosts_list[2], 'reservations': []}] non_free = [] plugin = instance_plugin.VirtualInstancePlugin() @@ -218,11 +218,12 @@ class TestVirtualInstancePlugin(tests.TestCase): 'memory_mb': 1024, 'disk_gb': 20, 'amount': 2, + 'affinity': False, 'resource_properties': '', 'start_date': datetime.datetime(2030, 1, 1, 8, 00), 'end_date': datetime.datetime(2030, 1, 1, 12, 00) } - expected = {'added': set(['host-2', 'host-3']), 'removed': set([])} + expected = {'added': ['host-2', 'host-3'], 'removed': []} ret = plugin.pickup_hosts('reservation-id1', values) self.assertEqual(expected, ret) @@ -260,11 +261,12 @@ class TestVirtualInstancePlugin(tests.TestCase): 'memory_mb': 1024, 'disk_gb': 20, 'amount': 2, + 'affinity': False, 'resource_properties': '', 'start_date': datetime.datetime(2030, 1, 1, 8, 00), 'end_date': datetime.datetime(2030, 1, 1, 12, 00), } - expected = {'added': set(['host-1', 'host-2']), 'removed': set([])} + expected = {'added': ['host-1', 'host-2'], 'removed': []} ret = plugin.pickup_hosts('reservation-id1', values) self.assertEqual(expected, ret) @@ -314,18 +316,123 @@ class TestVirtualInstancePlugin(tests.TestCase): 'memory_mb': 1024, 'disk_gb': 20, 'amount': 2, + 'affinity': False, 'resource_properties': '', 'start_date': datetime.datetime(2030, 1, 1, 8, 00), 'end_date': datetime.datetime(2030, 1, 1, 12, 00) } - expected = {'added': set(['host-1', 'host-3']), 'removed': set([])} + expected = {'added': ['host-1', 'host-3'], 'removed': []} ret = plugin.pickup_hosts('reservation-id1', params) self.assertEqual(expected, ret) expected_query = ['vcpus >= 1', 'memory_mb >= 1024', 'local_gb >= 20'] mock_host_get_query.assert_called_once_with(expected_query) + def test_pickup_host_with_affinity(self): + def fake_get_reservation_by_host(host_id, start, end): + if host_id in ['host-1', 'host-3']: + return [ + {'id': '1', + 'resource_type': instances.RESOURCE_TYPE}, + {'id': '2', + 'resource_type': instances.RESOURCE_TYPE} + ] + else: + return [] + + plugin = instance_plugin.VirtualInstancePlugin() + + mock_host_allocation_get = self.patch( + db_api, 'host_allocation_get_all_by_values') + mock_host_allocation_get.return_value = [] + + mock_host_get_query = self.patch(db_api, + 'reservable_host_get_all_by_queries') + hosts_list = [self.generate_host_info('host-1', 8, 8192, 1000), + self.generate_host_info('host-2', 2, 2048, 500), + self.generate_host_info('host-3', 2, 2048, 500)] + mock_host_get_query.return_value = hosts_list + + mock_get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + + mock_get_reservations.side_effect = fake_get_reservation_by_host + + mock_max_usages = self.patch(plugin, 'max_usages') + mock_max_usages.return_value = (0, 0, 0) + + mock_reservation_get = self.patch(db_api, 'reservation_get') + mock_reservation_get.return_value = { + 'status': 'pending' + } + + params = { + 'vcpus': 2, + 'memory_mb': 2048, + 'disk_gb': 100, + 'amount': 2, + 'affinity': True, + 'resource_properties': '', + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + + expected = {'added': ['host-1', 'host-1'], 'removed': []} + ret = plugin.pickup_hosts('reservation-id1', params) + + self.assertEqual(expected, ret) + expected_query = ['vcpus >= 2', 'memory_mb >= 2048', 'local_gb >= 100'] + mock_host_get_query.assert_called_once_with(expected_query) + + def test_pickup_host_with_no_affinity(self): + def fake_get_reservation_by_host(host_id, start, end): + return [] + + plugin = instance_plugin.VirtualInstancePlugin() + + mock_host_allocation_get = self.patch( + db_api, 'host_allocation_get_all_by_values') + mock_host_allocation_get.return_value = [] + + mock_host_get_query = self.patch(db_api, + 'reservable_host_get_all_by_queries') + hosts_list = [self.generate_host_info('host-1', 8, 8192, 1000), + self.generate_host_info('host-2', 2, 2048, 500), + self.generate_host_info('host-3', 2, 2048, 500)] + mock_host_get_query.return_value = hosts_list + + mock_get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + + mock_get_reservations.side_effect = fake_get_reservation_by_host + + mock_max_usages = self.patch(plugin, 'max_usages') + mock_max_usages.return_value = (0, 0, 0) + + mock_reservation_get = self.patch(db_api, 'reservation_get') + mock_reservation_get.return_value = { + 'status': 'pending' + } + + params = { + 'vcpus': 4, + 'memory_mb': 4096, + 'disk_gb': 200, + 'amount': 2, + 'affinity': None, + 'resource_properties': '', + 'start_date': datetime.datetime(2030, 1, 1, 8, 00), + 'end_date': datetime.datetime(2030, 1, 1, 12, 00) + } + + expected = {'added': ['host-1', 'host-1'], 'removed': []} + ret = plugin.pickup_hosts('reservation-id1', params) + + self.assertEqual(expected, ret) + expected_query = ['vcpus >= 4', 'memory_mb >= 4096', 'local_gb >= 200'] + mock_host_get_query.assert_called_once_with(expected_query) + def test_pickup_host_from_less_hosts(self): def fake_get_reservation_by_host(host_id, start, end): if host_id in ['host-1', 'host-3']: @@ -360,6 +467,21 @@ class TestVirtualInstancePlugin(tests.TestCase): db_api, 'host_allocation_get_all_by_values') mock_host_allocation_get.return_value = [] + old_reservation = { + 'id': 'reservation-id1', + 'status': 'pending', + 'lease_id': 'lease-id1', + 'resource_id': 'instance-reservation-id1', + 'vcpus': 2, 'memory_mb': 1024, 'disk_gb': 100, + 'amount': 2, 'affinity': False, + 'resource_properties': ''} + mock_reservation_get = self.patch(db_api, 'reservation_get') + mock_reservation_get.return_value = old_reservation + + mock_lease_get = self.patch(db_api, 'lease_get') + mock_lease_get.return_value = {'start_date': '2030-01-01 8:00', + 'end_date': '2030-01-01 12:00'} + mock_max_usages = self.patch(plugin, 'max_usages') mock_max_usages.return_value = (1, 1024, 100) @@ -368,14 +490,15 @@ class TestVirtualInstancePlugin(tests.TestCase): 'memory_mb': 1024, 'disk_gb': 20, 'amount': 2, + 'affinity': False, 'resource_properties': '', 'start_date': datetime.datetime(2030, 1, 1, 8, 00), 'end_date': datetime.datetime(2030, 1, 1, 12, 00) } - hosts = plugin.pickup_hosts('reservation-id1', values) - self.assertTrue((len(hosts['added']) - len(hosts['removed'])) - < values['amount']) + self.assertRaises(mgr_exceptions.NotEnoughHostsAvailable, + plugin.update_reservation, 'reservation-id1', + values) def test_max_usage_with_serial_reservation(self): def fake_event_get(sort_key, sort_dir, filters): @@ -573,6 +696,32 @@ class TestVirtualInstancePlugin(tests.TestCase): mock_create_reservation_class.assert_called_once_with( 'reservation-id1') + def test_query_available_hosts(self): + mock_host_get_query = self.patch(db_api, + 'reservable_host_get_all_by_queries') + host1, host2, host3 = (self.generate_host_info(host_id, 4, 4096, 1000) + for host_id in ['host-1', 'host-2', 'host-3']) + hosts_list = [host1, host2, host3] + mock_host_get_query.return_value = hosts_list + + get_reservations = self.patch(db_utils, + 'get_reservations_by_host_id') + get_reservations.return_value = [] + + plugin = instance_plugin.VirtualInstancePlugin() + + query_params = { + 'cpus': 1, 'memory': 1024, 'disk': 10, + 'resource_properties': '', + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0) + } + + ret = plugin.query_available_hosts(**query_params) + + expected = [host1] * 4 + [host2] * 4 + [host3] * 4 + self.assertEqual(expected, ret) + def test_pickup_hosts_for_update(self): reservation = {'id': 'reservation-id1', 'status': 'pending'} plugin = instance_plugin.VirtualInstancePlugin() @@ -595,8 +744,8 @@ class TestVirtualInstancePlugin(tests.TestCase): values = self.get_input_values(1, 1024, 10, 1, False, '2020-07-01 10:00', '2020-07-01 11:00', 'lease-1', '') - expect = {'added': set([]), - 'removed': set(['host-id1', 'host-id2', 'host-id3'])} + expect = {'added': [], + 'removed': ['host-id1', 'host-id2', 'host-id3']} ret = plugin.pickup_hosts(reservation['id'], values) self.assertEqual(expect['added'], ret['added']) self.assertEqual(2, len(ret['removed'])) @@ -614,7 +763,7 @@ class TestVirtualInstancePlugin(tests.TestCase): values = self.get_input_values(1, 1024, 10, 3, False, '2020-07-01 10:00', '2020-07-01 11:00', 'lease-1', '["==", "key1", "value1"]') - expect = {'added': set(['host-id4']), 'removed': set(['host-id1'])} + expect = {'added': ['host-id4'], 'removed': ['host-id1']} ret = plugin.pickup_hosts(reservation['id'], values) self.assertEqual(expect['added'], ret['added']) self.assertEqual(expect['removed'], ret['removed']) @@ -628,16 +777,15 @@ class TestVirtualInstancePlugin(tests.TestCase): mock_query_available.assert_called_with(**query_params) # case: new amount is greater than old amount + host_ids = ('host-id1', 'host-id2', 'host-id3', 'host-id4') mock_query_available.return_value = [ - self.generate_host_info('host-id1', 2, 2024, 1000), - self.generate_host_info('host-id2', 2, 2024, 1000), - self.generate_host_info('host-id3', 2, 2024, 1000), - self.generate_host_info('host-id4', 2, 2024, 1000)] + self.generate_host_info(host_id, 2, 2024, 1000) + for host_id in host_ids] values = self.get_input_values(1, 1024, 10, 4, False, '2020-07-01 10:00', '2020-07-01 11:00', 'lease-1', '') - expect = {'added': set(['host-id4']), 'removed': set([])} + expect = {'added': ['host-id4'], 'removed': []} ret = plugin.pickup_hosts(reservation['id'], values) self.assertEqual(expect['added'], ret['added']) self.assertEqual(expect['removed'], ret['removed']) @@ -650,6 +798,40 @@ class TestVirtualInstancePlugin(tests.TestCase): } mock_query_available.assert_called_with(**query_params) + # case: affinity is changed to True + mock_query_available.return_value = [ + self.generate_host_info(host_id, 8, 8192, 1000) + for host_id in host_ids * 8] + + values = self.get_input_values(1, 1024, 10, 4, True, + '2020-07-01 10:00', '2020-07-01 11:00', + 'lease-1', '') + ret = plugin.pickup_hosts(reservation['id'], values) + + # We don't care which host id (1-3) is picked up + # Just make sure the same host is returned three times in "added" + added = ret['added'] + self.assertEqual(3, len(added)) + self.assertEqual(1, len(set(added))) + self.assertIn(added[0], ('host-id1', 'host-id2', 'host-id3')) + + # and make sure the other two hosts are removed + removed = ret['removed'] + self.assertEqual(2, len(removed)) + self.assertEqual(2, len(set(removed))) + expect_removed = set(host_ids) - set(added) + for host_id in removed: + self.assertIn(host_id, expect_removed) + + query_params = { + 'cpus': 1, 'memory': 1024, 'disk': 10, + 'resource_properties': '', + 'start_date': '2020-07-01 10:00', + 'end_date': '2020-07-01 11:00', + 'excludes_res': ['reservation-id1'] + } + mock_query_available.assert_called_with(**query_params) + def test_update_resources(self): reservation = { 'id': 'reservation-id1', @@ -790,11 +972,21 @@ class TestVirtualInstancePlugin(tests.TestCase): mock_lease_get.return_value = {'start_date': '2020-07-07 18:00', 'end_date': '2020-07-07 19:00'} - mock_pickup_hosts = self.patch(plugin, 'pickup_hosts') - mock_pickup_hosts.return_value = { - 'added': set(), 'removed': set(['host-id2'])} + # Mock that we have at least two hosts for (2 vcpus + 100 disk_gb), + # but we have only one for (4 vcpus and 200 disk_gb) + mock_alloc_get = self.patch(db_api, + 'host_allocation_get_all_by_values') + mock_alloc_get.return_value = [{'compute_host_id': 'host-id1'}, + {'compute_host_id': 'host-id2'}] - new_values = {'vcpus': 4, 'disk_gb': 200} + mock_query_available = self.patch(plugin, 'query_available_hosts') + mock_query_available.return_value = [ + self.generate_host_info('host-id1', 4, 2048, 1000)] + + new_values = {'vcpus': 4, 'disk_gb': 200, + 'start_date': datetime.datetime(2020, 7, 7, 18, 0), + 'end_date': datetime.datetime(2020, 7, 7, 19, 0), + 'id': '00ee4f12-77c8-44d5-abca-06a543210a50'} self.assertRaises(mgr_exceptions.NotEnoughHostsAvailable, plugin.update_reservation, 'reservation-id1', new_values) diff --git a/blazar/tests/utils/test_plugins.py b/blazar/tests/utils/test_plugins.py index 56c175c2..426afa62 100644 --- a/blazar/tests/utils/test_plugins.py +++ b/blazar/tests/utils/test_plugins.py @@ -77,3 +77,25 @@ class TestPluginsUtils(tests.TestCase): self.assertRaises( manager_exceptions.MalformedRequirements, plugins_utils.convert_requirements, 'something') + + def test_list_difference(self): + old_list = [1, 1, 2, 3, 4, 4, 4, 5] + new_list = [1, 2, 3, 4, 7, 8, 8] + + result = plugins_utils.list_difference(old_list, new_list) + + to_remove = [1, 4, 4, 5] + to_add = [7, 8, 8] + + self.assertEqual((to_remove, to_add), result) + + def test_list_difference_empty(self): + old_list = [] + new_list = [1, 2, 2, 2, 3, 4, 7, 8, 8] + + result = plugins_utils.list_difference(old_list, new_list) + + to_remove = [] + to_add = [1, 2, 2, 2, 3, 4, 7, 8, 8] + + self.assertEqual((to_remove, to_add), result) diff --git a/blazar/utils/plugins.py b/blazar/utils/plugins.py index 9a0c6bb7..d7d821c8 100644 --- a/blazar/utils/plugins.py +++ b/blazar/utils/plugins.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy from oslo_serialization import jsonutils import six @@ -76,3 +77,27 @@ def _requirements_with_and_keyword(requirements): isinstance(requirements[0], six.string_types) and requirements[0] == 'and' and all(convert_requirements(x) for x in requirements[1:])) + + +def list_difference(list1, list2): + """Return a tuple that shows the differences between lists. + + Example: + list1 = [1, 1, 2, 3, 4, 4, 4, 5] # old list + list2 = [1, 2, 3, 4, 7, 8, 8] # new list + -> ([1, 4, 4, 5], [7, 8, 8]) # (to_remove, to_add) + + """ + def list_subtract(list_a, list_b): + result = copy.copy(list_a) + for value in list_b: + if value in result: + try: + result.remove(value) + except ValueError: + pass + return result + + result1 = list_subtract(list1, list2) + result2 = list_subtract(list2, list1) + return result1, result2