From a42f58de50c84235f649d957ac925780d365e3db Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Tue, 10 Oct 2017 14:30:03 +1300 Subject: [PATCH] Allow to skip API calls to Neutron in instance tables Now instance panel is sending API calls to Neutron to get the addresses info about network. It take some time until Nova network info cache is synced when IP address operation like floating IP association is made in Neutron. The API calls to Neutron exist from this reason. However, it retrieves a full list of port, so it can potentially leads to performance issues in large deployments. This commit adds a setting flag to control whether API calls to Neutron is used or skipped in the project instance table. This commits drops a call of servers_update_addresses() in the admin instance table. In the admin instance table there is no need to retrieve IP addresses from neutron because the main purpose of the admin panel is to see all instances and IP addresses in nova network info cache will be synced soon. Closes-Bug: #1722417 Co-Authored-By: Akihiro Motoki Change-Id: Ie16784eb6352f70ab644dc8b6ea03fc6a881d3f9 --- doc/source/configuration/settings.rst | 33 +++++++++++++++++++ .../dashboards/admin/instances/tests.py | 15 --------- .../dashboards/admin/instances/views.py | 9 ----- .../dashboards/project/instances/tests.py | 16 +++++++-- .../dashboards/project/instances/views.py | 10 ++++++ .../local/local_settings.py.example | 5 +++ ...nstance-ip-addresses-b9db6703d8b010c8.yaml | 10 ++++++ 7 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 releasenotes/notes/setting-retrieve-instance-ip-addresses-b9db6703d8b010c8.yaml diff --git a/doc/source/configuration/settings.rst b/doc/source/configuration/settings.rst index e01f30eb20..e564912705 100644 --- a/doc/source/configuration/settings.rst +++ b/doc/source/configuration/settings.rst @@ -2150,6 +2150,39 @@ Setting ``enable_quotas`` to ``False`` will make Horizon treat all Nova quotas as disabled, thus it won't try to modify them. By default, quotas are enabled. +OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 13.0.0(Queens) + +Default: ``True`` + +This settings controls whether IP addresses of servers are retrieved from +neutron in the project instance table. Setting this to ``False`` may mitigate +a performance issue in the project instance table in large deployments. + +If your deployment has no support of floating IP like provider network +scenario, you can set this to ``False`` in most cases. If your deployment +supports floating IP, read the detail below and understand the under-the-hood +before setting this to ``False``. + +Nova has a mechanism to cache network info but it is not fast enough +in some cases. For example, when a user associates a floating IP or +updates an IP address of an server port, it is not reflected to the nova +network info cache immediately. This means an action which a user makes +from the horizon instance table is not reflected into the table content +just after the action. To avoid this, horizon retrieves IP address info +from neutron when retrieving a list of servers from nova. + +On the other hand, this operation requires a full list of neutron ports +and can potentially lead to a performance issue in large deployments +(`bug 1722417 `__). +This issue can be avoided by skipping querying IP addresses to neutron +and setting this to ``False`` achieves this. +Note that when disabling the query to neutron it takes some time until +associated floating IPs are visible in the project instance table and +users may reload the table to check them. + OPENSTACK_NOVA_EXTENSIONS_BLACKLIST ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/openstack_dashboard/dashboards/admin/instances/tests.py b/openstack_dashboard/dashboards/admin/instances/tests.py index b35382187c..5a69852062 100644 --- a/openstack_dashboard/dashboards/admin/instances/tests.py +++ b/openstack_dashboard/dashboards/admin/instances/tests.py @@ -32,7 +32,6 @@ class InstanceViewTest(test.BaseAdminViewTests): @test.create_stubs({ api.nova: ('flavor_list', 'server_list', 'extension_supported',), api.keystone: ('tenant_list',), - api.network: ('servers_update_addresses',), api.glance: ('image_list_detailed',), }) def test_index(self): @@ -53,8 +52,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers, - all_tenants=True) self.mox.ReplayAll() res = self.client.get(INDEX_URL) @@ -66,7 +63,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova: ('flavor_list', 'flavor_get', 'server_list', 'extension_supported',), api.keystone: ('tenant_list',), - api.network: ('servers_update_addresses',), api.glance: ('image_list_detailed',), }) def test_index_flavor_list_exception(self): @@ -78,8 +74,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers, - all_tenants=True) api.nova.extension_supported('AdminActions', IsA(http.HttpRequest)) \ .MultipleTimes().AndReturn(True) api.nova.extension_supported('Shelve', IsA(http.HttpRequest)) \ @@ -103,7 +97,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova: ('flavor_list', 'flavor_get', 'server_list', 'extension_supported',), api.keystone: ('tenant_list',), - api.network: ('servers_update_addresses',), api.glance: ('image_list_detailed',), }) def test_index_flavor_get_exception(self): @@ -123,8 +116,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers, - all_tenants=True) api.nova.extension_supported('AdminActions', IsA(http.HttpRequest)) \ .MultipleTimes().AndReturn(True) api.nova.extension_supported('Shelve', IsA(http.HttpRequest)) \ @@ -207,7 +198,6 @@ class InstanceViewTest(test.BaseAdminViewTests): @test.create_stubs({ api.nova: ('flavor_list', 'server_list', 'extension_supported', ), api.keystone: ('tenant_list',), - api.network: ('servers_update_addresses',), api.glance: ('image_list_detailed',), }) def test_index_options_before_migrate(self): @@ -223,8 +213,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers, - all_tenants=True) api.nova.extension_supported('AdminActions', IsA(http.HttpRequest)) \ .MultipleTimes().AndReturn(True) api.nova.extension_supported('Shelve', IsA(http.HttpRequest)) \ @@ -239,7 +227,6 @@ class InstanceViewTest(test.BaseAdminViewTests): @test.create_stubs({ api.nova: ('flavor_list', 'server_list', 'extension_supported',), api.keystone: ('tenant_list',), - api.network: ('servers_update_addresses',), api.glance: ('image_list_detailed',), }) def test_index_options_after_migrate(self): @@ -263,8 +250,6 @@ class InstanceViewTest(test.BaseAdminViewTests): api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers, - all_tenants=True) self.mox.ReplayAll() res = self.client.get(INDEX_URL) diff --git a/openstack_dashboard/dashboards/admin/instances/views.py b/openstack_dashboard/dashboards/admin/instances/views.py index 2d3e53c08b..16120c3260 100644 --- a/openstack_dashboard/dashboards/admin/instances/views.py +++ b/openstack_dashboard/dashboards/admin/instances/views.py @@ -158,15 +158,6 @@ class AdminIndexView(tables.DataTableView): # don't call api.network return - try: - api.network.servers_update_addresses(self.request, instances, - all_tenants=True) - except Exception: - exceptions.handle( - self.request, - message=_('Unable to retrieve IP addresses from Neutron.'), - ignore=True) - with futurist.ThreadPoolExecutor(max_workers=3) as e: e.submit(fn=_task_get_tenants) e.submit(fn=_task_get_images) diff --git a/openstack_dashboard/dashboards/project/instances/tests.py b/openstack_dashboard/dashboards/project/instances/tests.py index 98b3d12d05..d4b5095bb9 100644 --- a/openstack_dashboard/dashboards/project/instances/tests.py +++ b/openstack_dashboard/dashboards/project/instances/tests.py @@ -132,7 +132,7 @@ class InstanceTests(helpers.ResetImageAPIVersionMixin, helpers.TestCase): 'servers_update_addresses', ), }) - def _get_index(self): + def _get_index(self, use_servers_update_address=True): servers = self.servers.list() api.nova.extension_supported('AdminActions', IsA(http.HttpRequest)) \ .MultipleTimes().AndReturn(True) @@ -148,7 +148,9 @@ class InstanceTests(helpers.ResetImageAPIVersionMixin, helpers.TestCase): search_opts = {'marker': None, 'paginate': True} api.nova.server_list(IsA(http.HttpRequest), search_opts=search_opts) \ .AndReturn([servers, False]) - api.network.servers_update_addresses(IsA(http.HttpRequest), servers) + if use_servers_update_address: + api.network.servers_update_addresses(IsA(http.HttpRequest), + servers) api.nova.tenant_absolute_limits(IsA(http.HttpRequest), reserved=True) \ .MultipleTimes().AndReturn(self.limits['absolute']) api.neutron.floating_ip_supported(IsA(http.HttpRequest)) \ @@ -170,6 +172,16 @@ class InstanceTests(helpers.ResetImageAPIVersionMixin, helpers.TestCase): self.assertItemsEqual(instances, self.servers.list()) self.assertNotContains(res, "Launch Instance (Quota exceeded)") + @override_settings(OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES=False) + def test_index_without_servers_update_addresses(self): + res = self._get_index(use_servers_update_address=False) + + self.assertTemplateUsed(res, INDEX_TEMPLATE) + instances = res.context['instances_table'].data + + self.assertItemsEqual(instances, self.servers.list()) + self.assertNotContains(res, "Launch Instance (Quota exceeded)") + @helpers.create_stubs({ api.nova: ('server_list', 'tenant_absolute_limits', 'flavor_list'), api.glance: ('image_list_detailed',), diff --git a/openstack_dashboard/dashboards/project/instances/views.py b/openstack_dashboard/dashboards/project/instances/views.py index 39a7b401aa..eb8761fdd5 100644 --- a/openstack_dashboard/dashboards/project/instances/views.py +++ b/openstack_dashboard/dashboards/project/instances/views.py @@ -91,6 +91,16 @@ class IndexView(tables.DataTableView): # don't call api.network return + # TODO(future): Explore more efficient logic to sync IP address + # and drop the setting OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES. + # The situation servers_update_addresses() is needed # is only + # when IP address of a server is updated via neutron API and + # nova network info cache is not synced. Precisely there is no + # need to check IP addresses of all serves. It is sufficient to + # fetch IP address information for servers recently updated. + if not getattr(settings, + 'OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES', True): + return try: api.network.servers_update_addresses(self.request, instances) except Exception: diff --git a/openstack_dashboard/local/local_settings.py.example b/openstack_dashboard/local/local_settings.py.example index 8f63dbe8bf..9ef441fb9e 100644 --- a/openstack_dashboard/local/local_settings.py.example +++ b/openstack_dashboard/local/local_settings.py.example @@ -303,6 +303,11 @@ OPENSTACK_HYPERVISOR_FEATURES = { 'enable_quotas': True } +# This settings controls whether IP addresses of servers are retrieved from +# neutron in the project instance table. Setting this to ``False`` may mitigate +# a performance issue in the project instance table in large deployments. +#OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES = True + # The OPENSTACK_CINDER_FEATURES settings can be used to enable optional # services provided by cinder that is not exposed by its extension API. OPENSTACK_CINDER_FEATURES = { diff --git a/releasenotes/notes/setting-retrieve-instance-ip-addresses-b9db6703d8b010c8.yaml b/releasenotes/notes/setting-retrieve-instance-ip-addresses-b9db6703d8b010c8.yaml new file mode 100644 index 0000000000..4a9e768bad --- /dev/null +++ b/releasenotes/notes/setting-retrieve-instance-ip-addresses-b9db6703d8b010c8.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + A new setting ``OPENSTACK_INSTANCE_RETRIEVE_IP_ADDRESSES`` was introduced + to control whether IP addresses of servers are retrieved from neutron in + the project instance table. This setting was introduces to mitigate a + performance issue in large deployments and setting this to ``False`` + skips the query to neutron. Deployments without floating IP support can + set this setting to ``False`` for better performance. + For more detail, see [:bug:`1722417`].