From c28d12604b60f2800fc2cadeba06408006cb5e69 Mon Sep 17 00:00:00 2001 From: Alexey Stepanov Date: Mon, 18 Apr 2016 14:36:04 +0300 Subject: [PATCH] Fix deprecated API usage of keystone client * Use keystoneauth1 for making session, which is shared throw all clients * KeystoneClient is now only wrapper over session Requirements has been updated to meet global requirements for mitaka release: 1. Bumped clients versions 2. requests has blacklisted versions due to bugs Closes-bug: #1571611 Change-Id: Icc59761b590b76a8d3ddac9b4f219efc097447a5 (cherry picked from commit 94170e0) --- fuelweb_test/helpers/common.py | 245 +++++++++++++++++------------ fuelweb_test/helpers/http.py | 57 ++++--- fuelweb_test/helpers/os_actions.py | 49 +++--- fuelweb_test/requirements.txt | 27 ++-- 4 files changed, 223 insertions(+), 155 deletions(-) diff --git a/fuelweb_test/helpers/common.py b/fuelweb_test/helpers/common.py index 44e4dd50c..3253e7961 100644 --- a/fuelweb_test/helpers/common.py +++ b/fuelweb_test/helpers/common.py @@ -12,21 +12,33 @@ # License for the specific language governing permissions and limitations # under the License. +import sys import time -from urlparse import urlparse +import traceback -from cinderclient import client as cinderclient +from cinderclient.client import Client as CinderClient from heatclient.v1.client import Client as HeatClient from glanceclient.v1 import Client as GlanceClient -import ironicclient.client as ironicclient +from ironicclient.client import Client as IronicClient +from keystoneauth1.exceptions import ClientException +from keystoneauth1.identity import V2Password +from keystoneauth1.session import Session as KeystoneSession from keystoneclient.v2_0 import Client as KeystoneClient -from keystoneclient.exceptions import ClientException -from novaclient.v2 import Client as NovaClient -import neutronclient.v2_0.client as neutronclient +from novaclient.client import Client as NovaClient +from neutronclient.v2_0.client import Client as NeutronClient from proboscis.asserts import assert_equal +import six +# pylint: disable=redefined-builtin +# noinspection PyUnresolvedReferences +from six.moves import xrange +# pylint: enable=redefined-builtin +# pylint: disable=import-error +# noinspection PyUnresolvedReferences +from six.moves import urllib +# pylint: enable=import-error -from fuelweb_test import logger as LOGGER -from fuelweb_test import logwrap as LOGWRAP +from fuelweb_test import logger +from fuelweb_test import logwrap from fuelweb_test.settings import DISABLE_SSL from fuelweb_test.settings import PATH_TO_CERT from fuelweb_test.settings import VERIFY_SSL @@ -35,14 +47,16 @@ from fuelweb_test.settings import VERIFY_SSL class Common(object): """Common.""" # TODO documentation + def __make_endpoint(self, endpoint): + parse = urllib.parse.urlparse(endpoint) + return parse._replace( + netloc='{}:{}'.format( + self.controller_ip, parse.port)).geturl() + def __init__(self, controller_ip, user, password, tenant): self.controller_ip = controller_ip - def make_endpoint(endpoint): - parse = urlparse(endpoint) - return parse._replace( - netloc='{}:{}'.format( - self.controller_ip, parse.port)).geturl() + self.keystone_session = None if DISABLE_SSL: auth_url = 'http://{0}:5000/v2.0/'.format(self.controller_ip) @@ -53,79 +67,100 @@ class Common(object): insecure = not VERIFY_SSL - LOGGER.debug('Auth URL is {0}'.format(auth_url)) + logger.debug('Auth URL is {0}'.format(auth_url)) - keystone_args = {'username': user, 'password': password, - 'tenant_name': tenant, 'auth_url': auth_url, - 'ca_cert': path_to_cert, 'insecure': insecure} - self.keystone = self._get_keystoneclient(**keystone_args) + self.__keystone_auth = V2Password( + auth_url=auth_url, + username=user, + password=password, + tenant_name=tenant) # TODO: in v3 project_name - token = self.keystone.auth_token - LOGGER.debug('Token is {0}'.format(token)) + self.__start_keystone_session(ca_cert=path_to_cert, insecure=insecure) - neutron_endpoint = self.keystone.service_catalog.url_for( - service_type='network', endpoint_type='publicURL') - neutron_args = {'username': user, 'password': password, - 'tenant_name': tenant, 'auth_url': auth_url, - 'ca_cert': path_to_cert, 'insecure': insecure, - 'endpoint_url': make_endpoint(neutron_endpoint)} - self.neutron = neutronclient.Client(**neutron_args) + @property + def keystone(self): + return KeystoneClient(session=self.keystone_session) - nova_endpoint = self.keystone.service_catalog.url_for( - service_type='compute', endpoint_type='publicURL') - nova_args = {'username': user, 'api_key': password, - 'project_id': tenant, 'auth_url': auth_url, - 'cacert': path_to_cert, 'insecure': insecure, - 'bypass_url': make_endpoint(nova_endpoint), - 'auth_token': token} - self.nova = NovaClient(**nova_args) + @property + def glance(self): + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='image')) + return GlanceClient( + session=self.keystone_session, + endpoint=endpoint, + endpoint_override=endpoint) - cinder_endpoint = self.keystone.service_catalog.url_for( - service_type='volume', endpoint_type='publicURL') - cinder_args = {'version': 1, 'username': user, - 'api_key': password, 'project_id': tenant, - 'auth_url': auth_url, 'cacert': path_to_cert, - 'insecure': insecure, - 'bypass_url': make_endpoint(cinder_endpoint)} - self.cinder = cinderclient.Client(**cinder_args) + @property + def neutron(self): + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='network')) + return NeutronClient( + session=self.keystone_session, + endpoint_override=endpoint) - glance_endpoint = self.keystone.service_catalog.url_for( - service_type='image', endpoint_type='publicURL') - LOGGER.debug('Glance endpoint is {0}'.format( - make_endpoint(glance_endpoint))) - glance_args = {'endpoint': make_endpoint(glance_endpoint), - 'token': token, - 'cacert': path_to_cert, - 'insecure': insecure} - self.glance = GlanceClient(**glance_args) + @property + def nova(self): + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='compute')) + return NovaClient( + version='2', + session=self.keystone_session, + endpoint_override=endpoint) - heat_endpoint = self.keystone.service_catalog.url_for( - service_type='orchestration', endpoint_type='publicURL') + @property + def cinder(self): + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='volume')) + return CinderClient( + version='1', + session=self.keystone_session, + endpoint_override=endpoint) - heat_args = {'endpoint': make_endpoint(heat_endpoint), - 'token': token, - 'cacert': path_to_cert, - 'insecure': insecure} - self.heat = HeatClient(**heat_args) + @property + def heat(self): + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='orchestration')) + return HeatClient( + session=self.keystone_session, + endpoint_override=endpoint) + @property + def ironic(self): try: - ironic_endpoint = self.keystone.service_catalog.url_for( - service_type='baremetal', - endpoint_type='publicURL') - self.ironic = ironicclient.get_client( - api_version=1, - os_auth_token=token, - ironic_url=make_endpoint(ironic_endpoint), insecure=True) + endpoint = self.__make_endpoint( + self._get_url_for_svc(service_type='baremetal')) + return IronicClient( + version='1', + session=self.keystone_session, + insecure=True, + endpoint_override=endpoint + ) except ClientException as e: - LOGGER.warning('Could not initialize ironic client {0}'.format(e)) + logger.warning('Could not initialize ironic client {0}'.format(e)) + raise + + @property + def keystone_access(self): + return self.__keystone_auth.get_access(session=self.keystone_session) + + def _get_url_for_svc( + self, service_type=None, interface='public', + region_name=None, service_name=None, + service_id=None, endpoint_id=None + ): + return self.keystone_access.service_catalog.url_for( + service_type=service_type, interface=interface, + region_name=region_name, service_name=service_name, + service_id=service_id, endpoint_id=endpoint_id + ) def goodbye_security(self): secgroup_list = self.nova.security_groups.list() - LOGGER.debug("Security list is {0}".format(secgroup_list)) + logger.debug("Security list is {0}".format(secgroup_list)) secgroup_id = [i.id for i in secgroup_list if i.name == 'default'][0] - LOGGER.debug("Id of security group default is {0}".format( + logger.debug("Id of security group default is {0}".format( secgroup_id)) - LOGGER.debug('Permit all TCP and ICMP in security group default') + logger.debug('Permit all TCP and ICMP in security group default') self.nova.security_group_rules.create(secgroup_id, ip_protocol='tcp', from_port=1, @@ -143,15 +178,16 @@ class Common(object): return self.glance.images.delete(image_id) def create_key(self, key_name): - LOGGER.debug('Try to create key {0}'.format(key_name)) + logger.debug('Try to create key {0}'.format(key_name)) return self.nova.keypairs.create(key_name) def create_instance(self, flavor_name='test_flavor', ram=64, vcpus=1, disk=1, server_name='test_instance', image_name=None, neutron_network=True, label=None): - LOGGER.debug('Try to create instance') + logger.debug('Try to create instance') start_time = time.time() + exc_type, exc_value, exc_traceback = None, None, None while time.time() - start_time < 100: try: if image_name: @@ -160,9 +196,13 @@ class Common(object): else: image = [i.id for i in self.nova.images.list()] break - except: - pass + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + logger.warning('Ignoring exception: {!r}'.format(e)) + logger.debug(traceback.format_exc()) else: + if all((exc_type, exc_traceback, exc_value)): + six.reraise(exc_type, exc_value, exc_traceback) raise Exception('Can not get image') kwargs = {} @@ -171,16 +211,16 @@ class Common(object): network = self.nova.networks.find(label=net_label) kwargs['nics'] = [{'net-id': network.id, 'v4-fixed-ip': ''}] - LOGGER.info('image uuid is {0}'.format(image)) + logger.info('image uuid is {0}'.format(image)) flavor = self.nova.flavors.create( name=flavor_name, ram=ram, vcpus=vcpus, disk=disk) - LOGGER.info('flavor is {0}'.format(flavor.name)) + logger.info('flavor is {0}'.format(flavor.name)) server = self.nova.servers.create( name=server_name, image=image[0], flavor=flavor, **kwargs) - LOGGER.info('server is {0}'.format(server.name)) + logger.info('server is {0}'.format(server.name)) return server - @LOGWRAP + @logwrap def get_instance_detail(self, server): details = self.nova.servers.get(server) return details @@ -193,13 +233,13 @@ class Common(object): try: _verify_instance_state() except AssertionError: - LOGGER.debug('Instance is not {0}, lets provide it the last ' + logger.debug('Instance is not {0}, lets provide it the last ' 'chance and sleep 60 sec'.format(expected_state)) time.sleep(60) _verify_instance_state() def delete_instance(self, server): - LOGGER.debug('Try to delete instance') + logger.debug('Try to delete instance') self.nova.servers.delete(server) def create_flavor(self, name, ram, vcpus, disk, flavorid="auto", @@ -211,29 +251,28 @@ class Common(object): def delete_flavor(self, flavor): return self.nova.flavors.delete(flavor) - def _get_keystoneclient(self, username, password, tenant_name, auth_url, - retries=3, ca_cert=None, insecure=False): - keystone = None - for i in range(retries): + def __start_keystone_session( + self, retries=3, ca_cert=None, insecure=not VERIFY_SSL): + exc_type, exc_value, exc_traceback = None, None, None + for i in xrange(retries): try: - if ca_cert: - keystone = KeystoneClient(username=username, - password=password, - tenant_name=tenant_name, - auth_url=auth_url, - cacert=ca_cert, - insecure=insecure) - + if insecure: + self.keystone_session = KeystoneSession( + auth=self.__keystone_auth, verify=False) + elif ca_cert: + self.keystone_session = KeystoneSession( + auth=self.__keystone_auth, verify=ca_cert) else: - keystone = KeystoneClient(username=username, - password=password, - tenant_name=tenant_name, - auth_url=auth_url) - break - except ClientException as e: - err = "Try nr {0}. Could not get keystone client, error: {1}" - LOGGER.warning(err.format(i + 1, e)) + self.keystone_session = KeystoneSession( + auth=self.__keystone_auth) + self.keystone_session.get_auth_headers() + return + + except ClientException as exc: + exc_type, exc_value, exc_traceback = sys.exc_info() + err = "Try nr {0}. Could not get keystone token, error: {1}" + logger.warning(err.format(i + 1, exc)) time.sleep(5) - if not keystone: - raise - return keystone + if exc_type and exc_traceback and exc_value: + six.reraise(exc_type, exc_value, exc_traceback) + raise RuntimeError() diff --git a/fuelweb_test/helpers/http.py b/fuelweb_test/helpers/http.py index 93451579e..0f7d7d03c 100644 --- a/fuelweb_test/helpers/http.py +++ b/fuelweb_test/helpers/http.py @@ -14,10 +14,19 @@ import json import traceback -import urllib2 +from keystoneauth1 import exceptions +from keystoneauth1.identity import V2Password +from keystoneauth1.session import Session as KeystoneSession from keystoneclient.v2_0 import Client as KeystoneClient -from keystoneclient import exceptions +# pylint: disable=import-error +# noinspection PyUnresolvedReferences +from six.moves.urllib import request +# noinspection PyUnresolvedReferences +from six.moves.urllib.error import HTTPError +# pylint: enable=import-error +import requests + from fuelweb_test import logger @@ -30,17 +39,21 @@ class HTTPClient(object): self.keystone_url = keystone_url self.creds = dict(credentials, **kwargs) self.keystone = None - self.opener = urllib2.build_opener(urllib2.HTTPHandler) + self.session = None + self.opener = request.build_opener(request.HTTPHandler) def authenticate(self): try: logger.info('Initialize keystoneclient with url %s', self.keystone_url) - self.keystone = KeystoneClient( - auth_url=self.keystone_url, **self.creds) - # it depends on keystone version, some versions doing auth - # explicitly some don't, but we are making it explicitly always - self.keystone.authenticate() + auth = V2Password( + auth_url=self.keystone_url, + username=self.creds['username'], + password=self.creds['password'], + tenant_name=self.creds['tenant_name']) + # TODO: in v3 project_name + self.session = KeystoneSession(auth=auth, verify=False) + self.keystone = KeystoneClient(session=self.session) logger.debug('Authorization token is successfully updated') except exceptions.AuthorizationFailure: logger.warning( @@ -51,7 +64,7 @@ class HTTPClient(object): def token(self): if self.keystone is not None: try: - return self.keystone.auth_token + return self.session.get_token() except exceptions.AuthorizationFailure: logger.warning( 'Cant establish connection to keystone with url %s', @@ -60,37 +73,45 @@ class HTTPClient(object): logger.warning("Keystone returned unauthorized error, trying " "to pass authentication.") self.authenticate() - return self.keystone.auth_token + return self.session.get_token() return None def get(self, endpoint): - req = urllib2.Request(self.url + endpoint) + req = request.Request(self.url + endpoint) return self._open(req) def post(self, endpoint, data=None, content_type="application/json"): if not data: data = {} - req = urllib2.Request(self.url + endpoint, data=json.dumps(data)) + req = request.Request(self.url + endpoint, data=json.dumps(data)) req.add_header('Content-Type', content_type) return self._open(req) def put(self, endpoint, data=None, content_type="application/json"): if not data: data = {} - req = urllib2.Request(self.url + endpoint, data=json.dumps(data)) + req = request.Request(self.url + endpoint, data=json.dumps(data)) req.add_header('Content-Type', content_type) req.get_method = lambda: 'PUT' return self._open(req) def delete(self, endpoint): - req = urllib2.Request(self.url + endpoint) + req = request.Request(self.url + endpoint) req.get_method = lambda: 'DELETE' return self._open(req) def _open(self, req): try: return self._get_response(req) - except urllib2.HTTPError as e: + except HTTPError as e: + if e.code == 308: + logger.info(e.read()) + url = req.get_full_url() + req = requests.get(url, headers={'X-Auth-Token': self.token}) + if req.status_code in [200]: + return req.json() + else: + req.raise_for_status() if e.code == 401: logger.warning('Authorization failure: {0}'.format(e.read())) self.authenticate() @@ -121,10 +142,10 @@ class HTTPClientZabbix(object): def __init__(self, url): self.url = url - self.opener = urllib2.build_opener(urllib2.HTTPHandler) + self.opener = request.build_opener(request.HTTPHandler) def get(self, endpoint=None, cookie=None): - req = urllib2.Request(self.url + endpoint) + req = request.Request(self.url + endpoint) if cookie: req.add_header('cookie', cookie) return self.opener.open(req) @@ -133,7 +154,7 @@ class HTTPClientZabbix(object): cookie=None): if not data: data = {} - req = urllib2.Request(self.url + endpoint, data=json.dumps(data)) + req = request.Request(self.url + endpoint, data=json.dumps(data)) req.add_header('Content-Type', content_type) if cookie: req.add_header('cookie', cookie) diff --git a/fuelweb_test/helpers/os_actions.py b/fuelweb_test/helpers/os_actions.py index a07af9bc7..ac5372d13 100644 --- a/fuelweb_test/helpers/os_actions.py +++ b/fuelweb_test/helpers/os_actions.py @@ -12,13 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -import paramiko -from proboscis import asserts import random import time from devops.error import TimeoutError from devops.helpers import helpers +import paramiko +from proboscis import asserts + from fuelweb_test.helpers import common from fuelweb_test import logger @@ -116,7 +117,7 @@ class OpenStackActions(common.Common): " is {0}".format(self.get_instance_detail(srv).status)) def create_server_for_migration(self, neutron=True, scenario='', - timeout=100, file=None, key_name=None, + timeout=100, filename=None, key_name=None, label=None, flavor=1, **kwargs): name = "test-serv" + str(random.randint(1, 0x7fffffff)) security_group = {} @@ -125,12 +126,12 @@ class OpenStackActions(common.Common): with open(scenario, "r+") as f: scenario = f.read() except Exception as exc: - logger.info("Error opening file: %s" % exc) + logger.info("Error opening file: {:s}".format(exc)) raise Exception() image_id = self._get_cirros_image().id - security_group[self.keystone.tenant_id] =\ + security_group[self.keystone_access.tenant_id] =\ self.create_sec_group_for_ssh() - security_group = [security_group[self.keystone.tenant_id].name] + security_groups = [security_group[self.keystone_access.tenant_id].name] if neutron: net_label = label if label else 'net04' @@ -138,15 +139,15 @@ class OpenStackActions(common.Common): if net.label == net_label] kwargs.update({'nics': [{'net-id': network[0]}], - 'security_groups': security_group}) + 'security_groups': security_groups}) else: - kwargs.update({'security_groups': security_group}) + kwargs.update({'security_groups': security_groups}) srv = self.nova.servers.create(name=name, image=image_id, flavor=flavor, userdata=scenario, - files=file, + files=filename, key_name=key_name, **kwargs) try: @@ -195,9 +196,8 @@ class OpenStackActions(common.Common): flip = self.neutron.create_floatingip(body) # Wait active state for port port_id = flip['floatingip']['port_id'] - helpers.wait( - lambda: - self.neutron.show_port(port_id)['port']['status'] == "ACTIVE") + helpers.wait(lambda: self.neutron.show_port( + port_id)['port']['status'] == "ACTIVE") return flip['floatingip'] fl_ips_pool = self.nova.floating_ip_pools.list() @@ -275,8 +275,9 @@ class OpenStackActions(common.Common): server = self.get_instance_detail(server.id) return server - def create_volume(self, size=1, image_id=None): - volume = self.cinder.volumes.create(size=size, imageRef=image_id) + def create_volume(self, size=1, image_id=None, **kwargs): + volume = self.cinder.volumes.create(size=size, imageRef=image_id, + **kwargs) helpers.wait( lambda: self.cinder.volumes.get(volume.id).status == "available", timeout=100) @@ -308,18 +309,19 @@ class OpenStackActions(common.Common): def get_hosts_for_migr(self, srv_host_name): # Determine which host is available for live migration - host_list = filter(lambda host: host.host_name != srv_host_name, - self.nova.hosts.list()) - return filter(lambda host: host._info['service'] == 'compute', - host_list) + return [ + host for host in self.nova.hosts.list() + if host.host_name != srv_host_name and + host._info['service'] == 'compute'] def get_md5sum(self, file_path, controller_ssh, vm_ip, creds=()): logger.info("Get file md5sum and compare it with previous one") out = self.execute_through_host( - controller_ssh, vm_ip, "md5sum %s" % file_path, creds) + controller_ssh, vm_ip, "md5sum {:s}".format(file_path), creds) return out['stdout'] - def execute_through_host(self, ssh, vm_host, cmd, creds=()): + @staticmethod + def execute_through_host(ssh, vm_host, cmd, creds=()): logger.debug("Making intermediate transport") intermediate_transport = ssh._ssh.get_transport() @@ -547,16 +549,17 @@ class OpenStackActions(common.Common): def get_vip(self, vip): return self.neutron.show_vip(vip) - def get_nova_instance_ip(self, srv, net_name='novanetwork', type='fixed'): + @staticmethod + def get_nova_instance_ip(srv, net_name='novanetwork', addrtype='fixed'): for network_label, address_list in srv.addresses.items(): if network_label != net_name: continue for addr in address_list: - if addr['OS-EXT-IPS:type'] == type: + if addr['OS-EXT-IPS:type'] == addrtype: return addr['addr'] raise Exception("Instance {0} doesn't have {1} address for network " "{2}, available addresses: {3}".format(srv.id, - type, + addrtype, net_name, srv.addresses)) diff --git a/fuelweb_test/requirements.txt b/fuelweb_test/requirements.txt index 7de5b7923..011d9a41c 100644 --- a/fuelweb_test/requirements.txt +++ b/fuelweb_test/requirements.txt @@ -4,18 +4,23 @@ paramiko>=1.16.0 # LGPL proboscis==1.2.6.0 ipaddr junitxml>=0.7.0 -netaddr -pyOpenSSL>=0.14 -python-glanceclient>=0.18.0 -python-keystoneclient>=0.3.2 -python-novaclient>=2.15.0 -python-cinderclient>=1.0.5 -python-neutronclient>=2.6.0 -python-ironicclient>=0.8.0 -python-heatclient>=0.6.0 +netaddr>=0.7.12,!=0.7.16 # BSD +pyOpenSSL>=0.14 # Apache-2.0 +Sphinx # BSD # Not required for tests, but required to build docs (pbr) +docutils # Not required for tests, but required to build docs (pbr) +markupsafe # Not required for tests, but required to build docs (pbr) +pytz>=2013.6 # MIT # Not required for tests, but required to build docs (pbr) +keystoneauth1>=2.1.0 # Apache-2.0 +python-glanceclient>=1.2.0 # Apache-2.0 +python-keystoneclient>=1.6.0,!=1.8.0,!=2.1.0 # Apache-2.0 +python-novaclient>=2.29.0,!=2.33.0 # Apache-2.0 +python-cinderclient>=1.3.1 # Apache-2.0 +python-neutronclient>=2.6.0,!=4.1.0 # Apache-2.0 +python-ironicclient>=1.1.0 # Apache-2.0 +python-heatclient>=0.6.0 # Apache-2.0 oslo.i18n>=3.1.0 -six -Jinja2 +six>=1.9.0 # MIT +Jinja2>=2.8 # BSD License (3 clause) AllPairs==2.0.1 launchpadlib beautifulsoup4>=4.2.0