From 445b4ace801848c3ae4a25e2cc01bd7e3c31f58d Mon Sep 17 00:00:00 2001 From: Xinyuan Huang Date: Mon, 6 Jul 2015 17:36:41 +0800 Subject: [PATCH] Modify aggregate disk/ram/vcpu constraints to use host state for aggregate info This removes db calls in them, and keeps them up-to-date with current nova. Change-Id: I3f06c07b2eda48b8cb234b60dca891bf2e2aab53 --- .../solvers/constraints/aggregate_disk.py | 23 ++++--------------- .../solvers/constraints/aggregate_ram.py | 23 ++++--------------- .../solvers/constraints/aggregate_vcpu.py | 23 ++++--------------- .../constraints/test_aggregate_disk.py | 18 +++++++++------ .../solvers/constraints/test_aggregate_ram.py | 18 +++++++++------ .../constraints/test_aggregate_vcpu.py | 18 +++++++++------ 6 files changed, 48 insertions(+), 75 deletions(-) diff --git a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_disk.py b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_disk.py index fa6293f..4d4ad50 100644 --- a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_disk.py +++ b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_disk.py @@ -16,9 +16,9 @@ from oslo_config import cfg from oslo_log import log as logging -from nova import db from nova.i18n import _LW from nova_solverscheduler.scheduler.solvers.constraints import disk_constraint +from nova_solverscheduler.scheduler.solvers import utils CONF = cfg.CONF CONF.import_opt('disk_allocation_ratio', 'nova.scheduler.filters.disk_filter') @@ -34,25 +34,12 @@ class AggregateDiskConstraint(disk_constraint.DiskConstraint): """ def _get_disk_allocation_ratio(self, host_state, filter_properties): - context = filter_properties['context'].elevated() - # TODO(uni): DB query in filter is a performance hit, especially for - # system with lots of hosts. Will need a general solution here to fix - # all filters with aggregate DB call things. - metadata = db.aggregate_metadata_get_by_host( - context, host_state.host, key='disk_allocation_ratio') - aggregate_vals = metadata.get('disk_allocation_ratio', set()) - num_values = len(aggregate_vals) - - if num_values == 0: - return CONF.disk_allocation_ratio - - if num_values > 1: - LOG.warn(_LW("%(num_values)d ratio values found, " - "of which the minimum value will be used."), - {'num_values': num_values}) + aggregate_vals = utils.aggregate_values_from_key( + host_state, 'disk_allocation_ratio') try: - ratio = min(map(float, aggregate_vals)) + ratio = utils.validate_num_values( + aggregate_vals, CONF.disk_allocation_ratio, cast_to=float) except ValueError as e: LOG.warning(_LW("Could not decode disk_allocation_ratio: '%s'"), e) ratio = CONF.disk_allocation_ratio diff --git a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_ram.py b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_ram.py index 563c9c7..a7e8383 100644 --- a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_ram.py +++ b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_ram.py @@ -16,9 +16,9 @@ from oslo_config import cfg from oslo_log import log as logging -from nova import db from nova.i18n import _LW from nova_solverscheduler.scheduler.solvers.constraints import ram_constraint +from nova_solverscheduler.scheduler.solvers import utils CONF = cfg.CONF CONF.import_opt('ram_allocation_ratio', 'nova.scheduler.filters.ram_filter') @@ -33,25 +33,12 @@ class AggregateRamConstraint(ram_constraint.RamConstraint): """ def _get_ram_allocation_ratio(self, host_state, filter_properties): - context = filter_properties['context'].elevated() - # TODO(uni): DB query in filter is a performance hit, especially for - # system with lots of hosts. Will need a general solution here to fix - # all filters with aggregate DB call things. - metadata = db.aggregate_metadata_get_by_host( - context, host_state.host, key='ram_allocation_ratio') - aggregate_vals = metadata.get('ram_allocation_ratio', set()) - num_values = len(aggregate_vals) - - if num_values == 0: - return CONF.ram_allocation_ratio - - if num_values > 1: - LOG.warn(_LW("%(num_values)d ratio values found, " - "of which the minimum value will be used."), - {'num_values': num_values}) + aggregate_vals = utils.aggregate_values_from_key( + host_state, 'ram_allocation_ratio') try: - ratio = min(map(float, aggregate_vals)) + ratio = utils.validate_num_values( + aggregate_vals, CONF.ram_allocation_ratio, cast_to=float) except ValueError as e: LOG.warning(_LW("Could not decode ram_allocation_ratio: '%s'"), e) ratio = CONF.ram_allocation_ratio diff --git a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_vcpu.py b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_vcpu.py index 0eb9230..7ee0c2a 100644 --- a/nova_solverscheduler/scheduler/solvers/constraints/aggregate_vcpu.py +++ b/nova_solverscheduler/scheduler/solvers/constraints/aggregate_vcpu.py @@ -16,9 +16,9 @@ from oslo_config import cfg from oslo_log import log as logging -from nova import db from nova.i18n import _LW from nova_solverscheduler.scheduler.solvers.constraints import vcpu_constraint +from nova_solverscheduler.scheduler.solvers import utils CONF = cfg.CONF CONF.import_opt('cpu_allocation_ratio', 'nova.scheduler.filters.core_filter') @@ -33,25 +33,12 @@ class AggregateVcpuConstraint(vcpu_constraint.VcpuConstraint): """ def _get_cpu_allocation_ratio(self, host_state, filter_properties): - context = filter_properties['context'].elevated() - # TODO(uni): DB query in filter is a performance hit, especially for - # system with lots of hosts. Will need a general solution here to fix - # all filters with aggregate DB call things. - metadata = db.aggregate_metadata_get_by_host( - context, host_state.host, key='cpu_allocation_ratio') - aggregate_vals = metadata.get('cpu_allocation_ratio', set()) - num_values = len(aggregate_vals) - - if num_values == 0: - return CONF.cpu_allocation_ratio - - if num_values > 1: - LOG.warning(_LW("%(num_values)d ratio values found, " - "of which the minimum value will be used."), - {'num_values': num_values}) + aggregate_vals = utils.aggregate_values_from_key( + host_state, 'cpu_allocation_ratio') try: - ratio = min(map(float, aggregate_vals)) + ratio = utils.validate_num_values( + aggregate_vals, CONF.cpu_allocation_ratio, cast_to=float) except ValueError as e: LOG.warning(_LW("Could not decode cpu_allocation_ratio: '%s'"), e) ratio = CONF.cpu_allocation_ratio diff --git a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_disk.py b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_disk.py index afd5dac..843b9ad 100644 --- a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_disk.py +++ b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_disk.py @@ -43,17 +43,18 @@ class TestAggregateDiskConstraint(test.NoDBTestCase): {'free_disk_mb': 1024, 'total_usable_disk_gb': 2}) self.fake_hosts = [host1, host2, host3] - @mock.patch('nova.db.aggregate_metadata_get_by_host') + @mock.patch('nova_solverscheduler.scheduler.solvers.utils.' + 'aggregate_values_from_key') def test_get_constraint_matrix(self, agg_mock): self.flags(disk_allocation_ratio=1.0) def _agg_mock_side_effect(*args, **kwargs): - if args[1] == 'host1': - return {'disk_allocation_ratio': set(['2.0', '3.0'])} - if args[1] == 'host2': - return {'disk_allocation_ratio': set(['3.0'])} - if args[1] == 'host3': - return {'disk_allocation_ratio': set()} + if args[0].host == 'host1': + return set(['2.0', '3.0']) + if args[0].host == 'host2': + return set(['3.0']) + if args[0].host == 'host3': + return set([]) agg_mock.side_effect = _agg_mock_side_effect expected_cons_mat = [ @@ -62,4 +63,7 @@ class TestAggregateDiskConstraint(test.NoDBTestCase): [False, False]] cons_mat = self.constraint_cls().get_constraint_matrix( self.fake_hosts, self.fake_filter_properties) + agg_mock.assert_any_call(self.fake_hosts[0], 'disk_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[1], 'disk_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[2], 'disk_allocation_ratio') self.assertEqual(expected_cons_mat, cons_mat) diff --git a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_ram.py b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_ram.py index c53c91a..2e7de0e 100644 --- a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_ram.py +++ b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_ram.py @@ -43,17 +43,18 @@ class TestAggregateRamConstraint(test.NoDBTestCase): {'free_ram_mb': 512, 'total_usable_ram_mb': 1024}) self.fake_hosts = [host1, host2, host3] - @mock.patch('nova.db.aggregate_metadata_get_by_host') + @mock.patch('nova_solverscheduler.scheduler.solvers.utils.' + 'aggregate_values_from_key') def test_get_constraint_matrix(self, agg_mock): self.flags(ram_allocation_ratio=1.0) def _agg_mock_side_effect(*args, **kwargs): - if args[1] == 'host1': - return {'ram_allocation_ratio': set(['1.0', '2.0'])} - if args[1] == 'host2': - return {'ram_allocation_ratio': set(['3.0'])} - if args[1] == 'host3': - return {'ram_allocation_ratio': set()} + if args[0].host == 'host1': + return set(['1.0', '2.0']) + if args[0].host == 'host2': + return set(['3.0']) + if args[0].host == 'host3': + return set([]) agg_mock.side_effect = _agg_mock_side_effect expected_cons_mat = [ @@ -62,4 +63,7 @@ class TestAggregateRamConstraint(test.NoDBTestCase): [False, False]] cons_mat = self.constraint_cls().get_constraint_matrix( self.fake_hosts, self.fake_filter_properties) + agg_mock.assert_any_call(self.fake_hosts[0], 'ram_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[1], 'ram_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[2], 'ram_allocation_ratio') self.assertEqual(expected_cons_mat, cons_mat) diff --git a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_vcpu.py b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_vcpu.py index bd2481b..0d847a2 100644 --- a/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_vcpu.py +++ b/nova_solverscheduler/tests/scheduler/solvers/constraints/test_aggregate_vcpu.py @@ -43,17 +43,18 @@ class TestAggregateVcpuConstraint(test.NoDBTestCase): {'vcpus_total': 16, 'vcpus_used': 16}) self.fake_hosts = [host1, host2, host3] - @mock.patch('nova.db.aggregate_metadata_get_by_host') + @mock.patch('nova_solverscheduler.scheduler.solvers.utils.' + 'aggregate_values_from_key') def test_get_constraint_matrix(self, agg_mock): self.flags(cpu_allocation_ratio=1.0) def _agg_mock_side_effect(*args, **kwargs): - if args[1] == 'host1': - return {'cpu_allocation_ratio': set(['1.0', '2.0'])} - if args[1] == 'host2': - return {'cpu_allocation_ratio': set(['3.0'])} - if args[1] == 'host3': - return {'cpu_allocation_ratio': set()} + if args[0].host == 'host1': + return set(['1.0', '2.0']) + if args[0].host == 'host2': + return set(['3.0']) + if args[0].host == 'host3': + return set([]) agg_mock.side_effect = _agg_mock_side_effect expected_cons_mat = [ @@ -62,4 +63,7 @@ class TestAggregateVcpuConstraint(test.NoDBTestCase): [False, False]] cons_mat = self.constraint_cls().get_constraint_matrix( self.fake_hosts, self.fake_filter_properties) + agg_mock.assert_any_call(self.fake_hosts[0], 'cpu_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[1], 'cpu_allocation_ratio') + agg_mock.assert_any_call(self.fake_hosts[2], 'cpu_allocation_ratio') self.assertEqual(expected_cons_mat, cons_mat)