From 986407ddd3e8cafac9e699ca90886ae8cb5c1bf9 Mon Sep 17 00:00:00 2001 From: Andrea Frittoli Date: Wed, 11 Oct 2017 10:23:17 +0000 Subject: [PATCH] Move the object client to tempest.lib Move the object client to tempest.lib and cleanup the last bits of code that were required to manage internal-only tempest service clients. Change-Id: Ia9aeab78e530c798dfa7b7d6f71e4b3ea3a84b7f --- ...as-stable-interface-d1b07c7e8f17bef6.yaml} | 1 + tempest/clients.py | 21 +--- tempest/cmd/verify_tempest_config.py | 4 +- tempest/config.py | 2 +- tempest/lib/services/clients.py | 27 +---- .../lib/services/object_storage/__init__.py | 25 ++++ .../services/object_storage/object_client.py | 0 tempest/services/object_storage/__init__.py | 25 ---- .../object_storage/test_object_client.py | 108 ++++++++++++++++++ .../tests/lib/services/registry_fixture.py | 4 +- tempest/tests/lib/services/test_clients.py | 5 +- tempest/tests/services/__init__.py | 0 .../tests/services/object_storage/__init__.py | 0 13 files changed, 149 insertions(+), 73 deletions(-) rename releasenotes/notes/{make-account-client-as-stable-interface-d1b07c7e8f17bef6.yaml => make-object-storage-client-as-stable-interface-d1b07c7e8f17bef6.yaml} (92%) rename tempest/{ => lib}/services/object_storage/object_client.py (100%) delete mode 100644 tempest/services/object_storage/__init__.py create mode 100644 tempest/tests/lib/services/object_storage/test_object_client.py delete mode 100644 tempest/tests/services/__init__.py delete mode 100644 tempest/tests/services/object_storage/__init__.py diff --git a/releasenotes/notes/make-account-client-as-stable-interface-d1b07c7e8f17bef6.yaml b/releasenotes/notes/make-object-storage-client-as-stable-interface-d1b07c7e8f17bef6.yaml similarity index 92% rename from releasenotes/notes/make-account-client-as-stable-interface-d1b07c7e8f17bef6.yaml rename to releasenotes/notes/make-object-storage-client-as-stable-interface-d1b07c7e8f17bef6.yaml index 5f899cf0a8..2bba952da5 100644 --- a/releasenotes/notes/make-account-client-as-stable-interface-d1b07c7e8f17bef6.yaml +++ b/releasenotes/notes/make-object-storage-client-as-stable-interface-d1b07c7e8f17bef6.yaml @@ -8,3 +8,4 @@ features: * account_client * container_client + * object_client diff --git a/tempest/clients.py b/tempest/clients.py index e617c3ce9f..0e346e0e79 100644 --- a/tempest/clients.py +++ b/tempest/clients.py @@ -17,7 +17,6 @@ from tempest import config from tempest.lib import auth from tempest.lib import exceptions as lib_exc from tempest.lib.services import clients -from tempest.services import object_storage CONF = config.CONF @@ -281,21 +280,11 @@ class Manager(clients.ServiceClients): self.snapshots_client_latest = self.snapshots_v3_client def _set_object_storage_clients(self): - # NOTE(andreaf) Load configuration from config. Once object storage - # is in lib, configuration will be pulled directly from the registry - # and this will not be required anymore. - params = config.service_client_config('object-storage') - - self.account_client = object_storage.AccountClient(self.auth_provider, - **params) - self.bulk_client = object_storage.BulkMiddlewareClient( - self.auth_provider, **params) - self.capabilities_client = object_storage.CapabilitiesClient( - self.auth_provider, **params) - self.container_client = object_storage.ContainerClient( - self.auth_provider, **params) - self.object_client = object_storage.ObjectClient(self.auth_provider, - **params) + self.account_client = self.object_storage.AccountClient() + self.bulk_client = self.object_storage.BulkMiddlewareClient() + self.capabilities_client = self.object_storage.CapabilitiesClient() + self.container_client = self.object_storage.ContainerClient() + self.object_client = self.object_storage.ObjectClient() def get_auth_provider_class(credentials): diff --git a/tempest/cmd/verify_tempest_config.py b/tempest/cmd/verify_tempest_config.py index a72493de0d..3fff9af841 100644 --- a/tempest/cmd/verify_tempest_config.py +++ b/tempest/cmd/verify_tempest_config.py @@ -76,7 +76,6 @@ from tempest.common import credentials_factory as credentials from tempest import config import tempest.lib.common.http from tempest.lib import exceptions as lib_exc -from tempest.services import object_storage CONF = config.CONF @@ -236,11 +235,10 @@ def verify_api_versions(os, service, update): def get_extension_client(os, service): - params = config.service_client_config('object-storage') extensions_client = { 'nova': os.compute.ExtensionsClient(), 'neutron': os.network.ExtensionsClient(), - 'swift': object_storage.CapabilitiesClient(os.auth_provider, **params), + 'swift': os.object_storage.CapabilitiesClient(), # NOTE: Cinder v3 API is current and v2 and v1 are deprecated. # V3 extension API is the same as v2, so we reuse the v2 client # for v3 API also. diff --git a/tempest/config.py b/tempest/config.py index 024a638580..7b4b2e7e70 100644 --- a/tempest/config.py +++ b/tempest/config.py @@ -1384,7 +1384,7 @@ def _register_tempest_service_clients(): module = service_clients[service_client] configs = service_client.split('.')[0] service_client_data = dict( - name=service_client.replace('.', '_'), + name=service_client.replace('.', '_').replace('-', '_'), service_version=service_client, module_path=module.__name__, client_names=module.__all__, diff --git a/tempest/lib/services/clients.py b/tempest/lib/services/clients.py index 4fa7a7a3cb..7071e4b6e4 100644 --- a/tempest/lib/services/clients.py +++ b/tempest/lib/services/clients.py @@ -31,6 +31,7 @@ from tempest.lib.services import compute from tempest.lib.services import identity from tempest.lib.services import image from tempest.lib.services import network +from tempest.lib.services import object_storage from tempest.lib.services import volume warnings.simplefilter("once") @@ -50,20 +51,13 @@ def tempest_modules(): 'image.v1': image.v1, 'image.v2': image.v2, 'network': network, + 'object-storage': object_storage, 'volume.v1': volume.v1, 'volume.v2': volume.v2, 'volume.v3': volume.v3 } -def _tempest_internal_modules(): - # Set of unstable service clients available in Tempest - # NOTE(andreaf) This list will exists only as long the remain clients - # are migrated to tempest.lib, and it will then be deleted without - # deprecation or advance notice - return set(['object-storage']) - - def available_modules(): """Set of service client modules available in Tempest and plugins @@ -101,17 +95,6 @@ def available_modules(): plug_service_versions)) name_conflicts.append(exceptions.PluginRegistrationException( name=plugin_name, detailed_error=detailed_error)) - # NOTE(andreaf) Once all tempest clients are stable, the following - # if will have to be removed. - if not plug_service_versions.isdisjoint( - _tempest_internal_modules()): - detailed_error = ( - 'Plugin %s is trying to register a service %s already ' - 'claimed by a Tempest one' % (plugin_name, - _tempest_internal_modules() & - plug_service_versions)) - name_conflicts.append(exceptions.PluginRegistrationException( - name=plugin_name, detailed_error=detailed_error)) extra_service_versions |= plug_service_versions if name_conflicts: LOG.error( @@ -369,7 +352,7 @@ class ServiceClients(object): client_parameters = client_parameters or {} self.parameters = {} # Parameters are provided for unversioned services - all_modules = available_modules() | _tempest_internal_modules() + all_modules = available_modules() unversioned_services = set( [x.split('.')[0] for x in all_modules]) for service in unversioned_services: @@ -456,9 +439,7 @@ class ServiceClients(object): @property def registered_services(self): - # NOTE(andreaf) Once all tempest modules are stable this needs to - # be updated to remove _tempest_internal_modules - return self._registered_services | _tempest_internal_modules() + return self._registered_services def _setup_parameters(self, parameters): """Setup default values for client parameters diff --git a/tempest/lib/services/object_storage/__init__.py b/tempest/lib/services/object_storage/__init__.py index e69de29bb2..4303d09e5b 100644 --- a/tempest/lib/services/object_storage/__init__.py +++ b/tempest/lib/services/object_storage/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +from tempest.lib.services.object_storage.account_client import AccountClient +from tempest.lib.services.object_storage.bulk_middleware_client import \ + BulkMiddlewareClient +from tempest.lib.services.object_storage.capabilities_client import \ + CapabilitiesClient +from tempest.lib.services.object_storage.container_client import \ + ContainerClient +from tempest.lib.services.object_storage.object_client import ObjectClient + +__all__ = ['AccountClient', 'BulkMiddlewareClient', 'CapabilitiesClient', + 'ContainerClient', 'ObjectClient'] diff --git a/tempest/services/object_storage/object_client.py b/tempest/lib/services/object_storage/object_client.py similarity index 100% rename from tempest/services/object_storage/object_client.py rename to tempest/lib/services/object_storage/object_client.py diff --git a/tempest/services/object_storage/__init__.py b/tempest/services/object_storage/__init__.py deleted file mode 100644 index 166e475df7..0000000000 --- a/tempest/services/object_storage/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -from tempest.lib.services.object_storage.account_client import AccountClient -from tempest.lib.services.object_storage.bulk_middleware_client import \ - BulkMiddlewareClient -from tempest.lib.services.object_storage.capabilities_client import \ - CapabilitiesClient -from tempest.lib.services.object_storage.container_client import \ - ContainerClient -from tempest.services.object_storage.object_client import ObjectClient - -__all__ = ['AccountClient', 'BulkMiddlewareClient', 'CapabilitiesClient', - 'ContainerClient', 'ObjectClient'] diff --git a/tempest/tests/lib/services/object_storage/test_object_client.py b/tempest/tests/lib/services/object_storage/test_object_client.py new file mode 100644 index 0000000000..a16d1d77ba --- /dev/null +++ b/tempest/tests/lib/services/object_storage/test_object_client.py @@ -0,0 +1,108 @@ +# Copyright 2016 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import mock + +from tempest.lib import exceptions +from tempest.lib.services.object_storage import object_client +from tempest.tests import base +from tempest.tests.lib import fake_auth_provider + + +class TestObjectClient(base.TestCase): + + def setUp(self): + super(TestObjectClient, self).setUp() + self.fake_auth = fake_auth_provider.FakeAuthProvider() + self.url = self.fake_auth.base_url(None) + self.object_client = object_client.ObjectClient(self.fake_auth, + 'swift', 'region1') + + @mock.patch.object(object_client, '_create_connection') + def test_create_object_continue_no_data(self, mock_poc): + self._validate_create_object_continue(None, mock_poc) + + @mock.patch.object(object_client, '_create_connection') + def test_create_object_continue_with_data(self, mock_poc): + self._validate_create_object_continue('hello', mock_poc) + + @mock.patch.object(object_client, '_create_connection') + def test_create_continue_with_no_continue_received(self, mock_poc): + self._validate_create_object_continue('hello', mock_poc, + initial_status=201) + + def _validate_create_object_continue(self, req_data, + mock_poc, initial_status=100): + + expected_hdrs = { + 'X-Auth-Token': self.fake_auth.get_token(), + 'content-length': 0 if req_data is None else len(req_data), + 'Expect': '100-continue'} + + # Setup the Mocks prior to invoking the object creation + mock_resp_cls = mock.Mock() + mock_resp_cls._read_status.return_value = ("1", initial_status, "OK") + + mock_poc.return_value.response_class.return_value = mock_resp_cls + + # This is the final expected return value + mock_poc.return_value.getresponse.return_value.status = 201 + mock_poc.return_value.getresponse.return_value.reason = 'OK' + + # Call method to PUT object using expect:100-continue + cnt = "container1" + obj = "object1" + path = "/%s/%s" % (cnt, obj) + + # If the expected initial status is not 100, then an exception + # should be thrown and the connection closed + if initial_status is 100: + status, reason = \ + self.object_client.create_object_continue(cnt, obj, req_data) + else: + self.assertRaises(exceptions.UnexpectedResponseCode, + self.object_client.create_object_continue, cnt, + obj, req_data) + mock_poc.return_value.close.assert_called_once_with() + + # Verify that putrequest is called 1 time with the appropriate values + mock_poc.return_value.putrequest.assert_called_once_with('PUT', path) + + # Verify that headers were written, including "Expect:100-continue" + calls = [] + + for header, value in expected_hdrs.items(): + calls.append(mock.call(header, value)) + + mock_poc.return_value.putheader.assert_has_calls(calls, False) + mock_poc.return_value.endheaders.assert_called_once_with() + + # The following steps are only taken if the initial status is 100 + if initial_status is 100: + # Verify that the method returned what it was supposed to + self.assertEqual(status, 201) + + # Verify that _safe_read was called once to remove the CRLF + # after the 100 response + mock_rc = mock_poc.return_value.response_class.return_value + mock_rc._safe_read.assert_called_once_with(2) + + # Verify the actual data was written via send + mock_poc.return_value.send.assert_called_once_with(req_data) + + # Verify that the getresponse method was called to receive + # the final + mock_poc.return_value.getresponse.assert_called_once_with() diff --git a/tempest/tests/lib/services/registry_fixture.py b/tempest/tests/lib/services/registry_fixture.py index 8484209222..1da2112d3d 100644 --- a/tempest/tests/lib/services/registry_fixture.py +++ b/tempest/tests/lib/services/registry_fixture.py @@ -38,7 +38,7 @@ class RegistryFixture(fixtures.Fixture): """Initialise the registry fixture""" self.services = set(['compute', 'identity.v2', 'identity.v3', 'image.v1', 'image.v2', 'network', 'volume.v1', - 'volume.v2', 'volume.v3']) + 'volume.v2', 'volume.v3', 'object-storage']) def _setUp(self): # Cleanup the registry @@ -50,7 +50,7 @@ class RegistryFixture(fixtures.Fixture): for sc in self.services: sc_module = service_clients[sc] sc_unversioned = sc.split('.')[0] - sc_name = sc.replace('.', '_') + sc_name = sc.replace('.', '_').replace('-', '_') # Pass the bare minimum params to satisfy the clients interface service_client_data = dict( name=sc_name, service_version=sc, service=sc_unversioned, diff --git a/tempest/tests/lib/services/test_clients.py b/tempest/tests/lib/services/test_clients.py index 6d0f27a602..43fd88ff61 100644 --- a/tempest/tests/lib/services/test_clients.py +++ b/tempest/tests/lib/services/test_clients.py @@ -189,9 +189,7 @@ class TestServiceClients(base.TestCase): def setUp(self): super(TestServiceClients, self).setUp() self.useFixture(fixtures.MockPatch( - 'tempest.lib.services.clients.tempest_modules', return_value={})) - self.useFixture(fixtures.MockPatch( - 'tempest.lib.services.clients._tempest_internal_modules', + 'tempest.lib.services.clients.tempest_modules', return_value=set(['fake_service1']))) def test___init___creds_v2_uri(self): @@ -416,6 +414,7 @@ class TestServiceClients(base.TestCase): _manager = self._get_manager() duplicate_service = 'fake_service1' expected_error = '.*' + duplicate_service + _manager._registered_services = [duplicate_service] with testtools.ExpectedException( exceptions.ServiceClientRegistrationException, expected_error): _manager.register_service_client_module( diff --git a/tempest/tests/services/__init__.py b/tempest/tests/services/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tempest/tests/services/object_storage/__init__.py b/tempest/tests/services/object_storage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000