From 96f10711667603e7fbad57b151c6438cdd9ae270 Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Tue, 20 Feb 2018 09:16:03 -0800 Subject: [PATCH] Honor availability_zone hint via placement This adds a request filter that, if enabled, allows us to use placement to select hosts in the desired availability zone by looking up the uuid of the associated host aggregate and using that in our query for allocation candidates. The deployer needs the same sort of mirrored aggregate setup as the tenant filter, and documentation is added here to make that clear. Related to blueprint placement-req-filter Change-Id: I7eb6de435e10793f5445724d847a8f1bf25ec6e3 --- doc/source/admin/configuration/schedulers.rst | 48 ++++++ nova/conf/scheduler.py | 16 ++ nova/scheduler/request_filter.py | 25 +++ nova/tests/functional/integrated_helpers.py | 5 +- nova/tests/functional/test_aggregates.py | 158 +++++++++++++++--- .../unit/scheduler/test_request_filter.py | 69 ++++++++ ...one-placement-filter-0006c9895853c9bc.yaml | 10 ++ 7 files changed, 303 insertions(+), 28 deletions(-) create mode 100644 releasenotes/notes/availability-zone-placement-filter-0006c9895853c9bc.yaml diff --git a/doc/source/admin/configuration/schedulers.rst b/doc/source/admin/configuration/schedulers.rst index 74d2abb624f5..3c2b5d633599 100644 --- a/doc/source/admin/configuration/schedulers.rst +++ b/doc/source/admin/configuration/schedulers.rst @@ -1239,6 +1239,54 @@ this function, and is enabled by setting $ openstack --os-placement-api-version=1.2 resource provider aggregate set --aggregate 019e2189-31b3-49e1-aff2-b220ebd91c24 815a5634-86fb-4e1e-8824-8a631fee3e06 +Availability Zones with Placement +--------------------------------- + +In order to use placement to honor availability zone requests, there must be +placement aggregates that match the membership and UUID of nova host aggregates +that you assign as availability zones. The same key in aggregate metadata used +by the `AvailabilityZoneFilter` filter controls this function, and is enabled by +setting `[scheduler]/query_placement_for_availability_zone=True`. + +.. code-block:: console + + $ openstack --os-compute-api-version=2.53 aggregate create myaz + +-------------------+--------------------------------------+ + | Field | Value | + +-------------------+--------------------------------------+ + | availability_zone | None | + | created_at | 2018-03-29T16:22:23.175884 | + | deleted | False | + | deleted_at | None | + | id | 4 | + | name | myaz | + | updated_at | None | + | uuid | 019e2189-31b3-49e1-aff2-b220ebd91c24 | + +-------------------+--------------------------------------+ + + $ openstack --os-compute-api-version=2.53 aggregate add host myaz node1 + +-------------------+--------------------------------------+ + | Field | Value | + +-------------------+--------------------------------------+ + | availability_zone | None | + | created_at | 2018-03-29T16:22:23.175884 | + | deleted | False | + | deleted_at | None | + | hosts | [u'node1'] | + | id | 4 | + | name | myagg | + | updated_at | None | + | uuid | 019e2189-31b3-49e1-aff2-b220ebd91c24 | + +-------------------+--------------------------------------+ + + $ openstack aggregate set --property availability_zone=az002 myaz + + $ openstack --os-placement-api-version=1.2 resource provider aggregate set --aggregate 019e2189-31b3-49e1-aff2-b220ebd91c24 815a5634-86fb-4e1e-8824-8a631fee3e06 + +With the above configuration, the `AvailabilityZoneFilter` filter can be disabled +in `[filter_scheduler]/enabled_filters` while retaining proper behavior (and doing +so with the higher performance of placement's implementation). + XenServer hypervisor pools to support live migration ---------------------------------------------------- diff --git a/nova/conf/scheduler.py b/nova/conf/scheduler.py index 5b20dacd2377..e2f3ef37a8fa 100644 --- a/nova/conf/scheduler.py +++ b/nova/conf/scheduler.py @@ -163,6 +163,22 @@ aggregate, then this should be True to prevent them from receiving unrestricted scheduling to any available node. See also the limit_tenants_to_placement_aggregate option. +"""), + cfg.BoolOpt("query_placement_for_availability_zone", + default=False, + help=""" +This setting causes the scheduler to look up a host aggregate with the +metadata key of `availability_zone` set to the value provided by an +incoming request, and request results from placement be limited to that +aggregate. + +The matching aggregate UUID must be mirrored in placement for proper +operation. If no host aggregate with the `availability_zone` key is +found, or that aggregate does not match one in placement, the result will +be the same as not finding any suitable hosts. + +Note that if you enable this flag, you can disable the (less efficient) +AvailabilityZoneFilter in the scheduler. """), ] diff --git a/nova/scheduler/request_filter.py b/nova/scheduler/request_filter.py index cbf81a724a2e..315251628012 100644 --- a/nova/scheduler/request_filter.py +++ b/nova/scheduler/request_filter.py @@ -60,8 +60,33 @@ def require_tenant_aggregate(ctxt, request_spec): reason=_('No hosts available for tenant')) +def map_az_to_placement_aggregate(ctxt, request_spec): + """Map requested nova availability zones to placement aggregates. + + This will modify request_spec to request hosts in an aggregate that + matches the desired AZ of the user's request. + """ + if not CONF.scheduler.query_placement_for_availability_zone: + return + + az_hint = request_spec.availability_zone + if not az_hint: + return + + aggregates = objects.AggregateList.get_by_metadata(ctxt, + key='availability_zone', + value=az_hint) + if aggregates: + if ('requested_destination' not in request_spec or + request_spec.requested_destination is None): + request_spec.requested_destination = objects.Destination() + request_spec.requested_destination.require_aggregates( + [agg.uuid for agg in aggregates]) + + ALL_REQUEST_FILTERS = [ require_tenant_aggregate, + map_az_to_placement_aggregate, ] diff --git a/nova/tests/functional/integrated_helpers.py b/nova/tests/functional/integrated_helpers.py index 8e803ec6c05d..941a4015e8d3 100644 --- a/nova/tests/functional/integrated_helpers.py +++ b/nova/tests/functional/integrated_helpers.py @@ -251,7 +251,8 @@ class InstanceHelperMixin(object): admin_api, server, {'status': expected_status}, max_retries) def _build_minimal_create_server_request(self, api, name, image_uuid=None, - flavor_id=None, networks=None): + flavor_id=None, networks=None, + az=None): server = {} # We now have a valid imageId @@ -264,6 +265,8 @@ class InstanceHelperMixin(object): server['name'] = name if networks is not None: server['networks'] = networks + if az is not None: + server['availability_zone'] = az return server def _wait_until_deleted(self, server): diff --git a/nova/tests/functional/test_aggregates.py b/nova/tests/functional/test_aggregates.py index 14b3a69250eb..f3e76cb3ccdd 100644 --- a/nova/tests/functional/test_aggregates.py +++ b/nova/tests/functional/test_aggregates.py @@ -14,7 +14,9 @@ import time from nova.scheduler.client import report +import nova.conf from nova import context as nova_context +from nova.scheduler import weights from nova import test from nova.tests import fixtures as nova_fixtures from nova.tests.functional import integrated_helpers @@ -22,6 +24,8 @@ import nova.tests.unit.image.fake from nova.tests.unit import policy_fixture from nova.virt import fake +CONF = nova.conf.CONF + class AggregatesTest(integrated_helpers._IntegratedTestBase): api_major_version = 'v2' @@ -101,12 +105,6 @@ class AggregateRequestFiltersTest(test.TestCase, # Aggregate with neither host self._create_aggregate('no-hosts') - # Default to enabling the filter and making it mandatory - self.flags(limit_tenants_to_placement_aggregate=True, - group='scheduler') - self.flags(placement_aggregate_required_for_tenants=True, - group='scheduler') - def _start_compute(self, host): """Start a nova compute service on the given host @@ -158,6 +156,46 @@ class AggregateRequestFiltersTest(test.TestCase, self.report_client.set_aggregates_for_provider(self.context, host_uuid, placement_aggs) + def _wait_for_state_change(self, server, from_status): + for i in range(0, 50): + server = self.api.get_server(server['id']) + if server['status'] != from_status: + break + time.sleep(.1) + + return server + + def _boot_server(self, az=None): + server_req = self._build_minimal_create_server_request( + self.api, 'test-instance', flavor_id=self.flavors[0]['id'], + image_uuid='155d900f-4e14-4e4c-a73d-069cbf4541e6', + networks='none', az=az) + + created_server = self.api.post_server({'server': server_req}) + server = self._wait_for_state_change(created_server, 'BUILD') + + return server + + def _get_instance_host(self, server): + srv = self.admin_api.get_server(server['id']) + return srv['OS-EXT-SRV-ATTR:host'] + + def _set_az_aggregate(self, agg, az): + """Set the availability_zone of an aggregate + + :param agg: Name of the nova aggregate + :param az: Availability zone name + """ + agg = self.aggregates[agg] + action = { + 'set_metadata': { + 'metadata': { + 'availability_zone': az, + } + }, + } + self.admin_api.post_aggregate_action(agg['id'], action) + def _grant_tenant_aggregate(self, agg, tenants): """Grant a set of tenants access to use an aggregate. @@ -175,25 +213,16 @@ class AggregateRequestFiltersTest(test.TestCase, } self.admin_api.post_aggregate_action(agg['id'], action) - def _wait_for_state_change(self, server, from_status): - for i in range(0, 50): - server = self.api.get_server(server['id']) - if server['status'] != from_status: - break - time.sleep(.1) - return server +class TenantAggregateFilterTest(AggregateRequestFiltersTest): + def setUp(self): + super(TenantAggregateFilterTest, self).setUp() - def _boot_server(self): - server_req = self._build_minimal_create_server_request( - self.api, 'test-instance', flavor_id=self.flavors[0]['id'], - image_uuid='155d900f-4e14-4e4c-a73d-069cbf4541e6', - networks='none') - - created_server = self.api.post_server({'server': server_req}) - server = self._wait_for_state_change(created_server, 'BUILD') - - return server + # Default to enabling the filter and making it mandatory + self.flags(limit_tenants_to_placement_aggregate=True, + group='scheduler') + self.flags(placement_aggregate_required_for_tenants=True, + group='scheduler') def test_tenant_id_required_fails_if_no_aggregate(self): server = self._boot_server() @@ -209,10 +238,6 @@ class AggregateRequestFiltersTest(test.TestCase, # creates should still succeed since aggregates are not required self.assertEqual('ACTIVE', server['status']) - def _get_instance_host(self, server): - srv = self.admin_api.get_server(server['id']) - return srv['OS-EXT-SRV-ATTR:host'] - def test_filter_honors_tenant_id(self): tenant = self.api.project_id @@ -268,3 +293,82 @@ class AggregateRequestFiltersTest(test.TestCase, server = self._boot_server() self.assertEqual('ACTIVE', server['status']) self.assertEqual('host2', self._get_instance_host(server)) + + +class HostNameWeigher(weights.BaseHostWeigher): + def _weigh_object(self, host_state, weight_properties): + """Arbitrary preferring host1 over host2 over host3.""" + weights = {'host1': 100, 'host2': 50, 'host3': 1} + return weights.get(host_state.host, 0) + + +class AvailabilityZoneFilterTest(AggregateRequestFiltersTest): + def setUp(self): + # Default to enabling the filter + self.flags(query_placement_for_availability_zone=True, + group='scheduler') + + # Use our custom weigher defined above to make sure that we have + # a predictable scheduling sort order. + self.flags(weight_classes=[__name__ + '.HostNameWeigher'], + group='filter_scheduler') + + # NOTE(danms): Do this before calling setUp() so that + # the scheduler service that is started sees the new value + filters = CONF.filter_scheduler.enabled_filters + filters.remove('AvailabilityZoneFilter') + self.flags(enabled_filters=filters, group='filter_scheduler') + + super(AvailabilityZoneFilterTest, self).setUp() + + def test_filter_with_az(self): + self._set_az_aggregate('only-host2', 'myaz') + server1 = self._boot_server(az='myaz') + server2 = self._boot_server(az='myaz') + hosts = [self._get_instance_host(s) for s in (server1, server2)] + self.assertEqual(['host2', 'host2'], hosts) + + +class TestAggregateFiltersTogether(AggregateRequestFiltersTest): + def setUp(self): + # NOTE(danms): Do this before calling setUp() so that + # the scheduler service that is started sees the new value + filters = CONF.filter_scheduler.enabled_filters + filters.remove('AvailabilityZoneFilter') + self.flags(enabled_filters=filters, group='filter_scheduler') + + super(TestAggregateFiltersTogether, self).setUp() + + # Default to enabling both filters + self.flags(limit_tenants_to_placement_aggregate=True, + group='scheduler') + self.flags(placement_aggregate_required_for_tenants=True, + group='scheduler') + self.flags(query_placement_for_availability_zone=True, + group='scheduler') + + def test_tenant_with_az_match(self): + # Grant our tenant access to the aggregate with + # host1 + self._grant_tenant_aggregate('only-host1', + [self.api.project_id]) + # Set an az on only-host1 + self._set_az_aggregate('only-host1', 'myaz') + + # Boot the server into that az and make sure we land + server = self._boot_server(az='myaz') + self.assertEqual('host1', self._get_instance_host(server)) + + def test_tenant_with_az_mismatch(self): + # Grant our tenant access to the aggregate with + # host1 + self._grant_tenant_aggregate('only-host1', + [self.api.project_id]) + # Set an az on only-host2 + self._set_az_aggregate('only-host2', 'myaz') + + # Boot the server into that az and make sure we fail + server = self._boot_server(az='myaz') + self.assertIsNone(self._get_instance_host(server)) + server = self.api.get_server(server['id']) + self.assertEqual('ERROR', server['status']) diff --git a/nova/tests/unit/scheduler/test_request_filter.py b/nova/tests/unit/scheduler/test_request_filter.py index f9a81c5e4d58..c3f5c27d3c47 100644 --- a/nova/tests/unit/scheduler/test_request_filter.py +++ b/nova/tests/unit/scheduler/test_request_filter.py @@ -27,6 +27,8 @@ class TestRequestFilter(test.NoDBTestCase): project_id=uuids.project) self.flags(limit_tenants_to_placement_aggregate=True, group='scheduler') + self.flags(query_placement_for_availability_zone=True, + group='scheduler') def test_process_reqspec(self): fake_filters = [mock.MagicMock(), mock.MagicMock()] @@ -83,3 +85,70 @@ class TestRequestFilter(test.NoDBTestCase): getmd.return_value = [] request_filter.require_tenant_aggregate( self.context, mock.MagicMock()) + + @mock.patch('nova.objects.AggregateList.get_by_metadata') + def test_map_az(self, getmd): + getmd.return_value = [objects.Aggregate(uuid=uuids.agg1)] + reqspec = objects.RequestSpec(availability_zone='fooaz') + request_filter.map_az_to_placement_aggregate(self.context, reqspec) + self.assertEqual([uuids.agg1], + reqspec.requested_destination.aggregates) + + @mock.patch('nova.objects.AggregateList.get_by_metadata') + def test_map_az_no_hint(self, getmd): + reqspec = objects.RequestSpec(availability_zone=None) + request_filter.map_az_to_placement_aggregate(self.context, reqspec) + self.assertNotIn('requested_destination', reqspec) + self.assertFalse(getmd.called) + + @mock.patch('nova.objects.AggregateList.get_by_metadata') + def test_map_az_no_aggregates(self, getmd): + getmd.return_value = [] + reqspec = objects.RequestSpec(availability_zone='fooaz') + request_filter.map_az_to_placement_aggregate(self.context, reqspec) + self.assertNotIn('requested_destination', reqspec) + getmd.assert_called_once_with(self.context, key='availability_zone', + value='fooaz') + + @mock.patch('nova.objects.AggregateList.get_by_metadata') + def test_map_az_disabled(self, getmd): + self.flags(query_placement_for_availability_zone=False, + group='scheduler') + reqspec = objects.RequestSpec(availability_zone='fooaz') + request_filter.map_az_to_placement_aggregate(self.context, reqspec) + getmd.assert_not_called() + + @mock.patch('nova.objects.AggregateList.get_by_metadata') + def test_with_tenant_and_az(self, getmd): + getmd.side_effect = [ + # Tenant filter + [objects.Aggregate( + uuid=uuids.agg1, + metadata={'filter_tenant_id': 'owner'}), + objects.Aggregate( + uuid=uuids.agg2, + metadata={'filter_tenant_id:12': 'owner'}), + objects.Aggregate( + uuid=uuids.agg3, + metadata={'other_key': 'owner'})], + # AZ filter + [objects.Aggregate( + uuid=uuids.agg4, + metadata={'availability_zone': 'myaz'})], + ] + reqspec = objects.RequestSpec(project_id='owner', + availability_zone='myaz') + request_filter.process_reqspec(self.context, reqspec) + self.assertEqual( + ','.join(sorted([uuids.agg1, uuids.agg2])), + ','.join(sorted( + reqspec.requested_destination.aggregates[0].split(',')))) + self.assertEqual( + ','.join(sorted([uuids.agg4])), + ','.join(sorted( + reqspec.requested_destination.aggregates[1].split(',')))) + getmd.assert_has_calls([ + mock.call(self.context, value='owner'), + mock.call(self.context, + key='availability_zone', + value='myaz')]) diff --git a/releasenotes/notes/availability-zone-placement-filter-0006c9895853c9bc.yaml b/releasenotes/notes/availability-zone-placement-filter-0006c9895853c9bc.yaml new file mode 100644 index 000000000000..c78e2e5e014a --- /dev/null +++ b/releasenotes/notes/availability-zone-placement-filter-0006c9895853c9bc.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The scheduler can now use placement to more efficiently query for hosts within + an availability zone. This requires that a host aggregate is created in nova + with the ``availability_zone`` key set, and the same aggregate is created in + placement with an identical UUID. The + ``[scheduler]/query_placement_for_availability_zone`` config option enables + this behavior and, if enabled, eliminates the need for the + ``AvailabilityZoneFilter`` to be enabled. \ No newline at end of file