Migrate to keystoneauth1 and cinder v2 APIs

Recheck TLS support with Newton target

Change-Id: I5625840a0373c7d03e1d0318648249dfea968871

kloudbuster/credentials.py

@@ -1,4 +1,4 @@
-# Copyright 2014 Cisco Systems, Inc.  All rights reserved.
+# Copyright 2016 Cisco Systems, Inc.  All rights reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,6 +15,9 @@
 
 # Module for credentials in Openstack
 import getpass
+from keystoneauth1.identity import v2
+from keystoneauth1.identity import v3
+from keystoneauth1 import session
 import os
 import re
 
@@ -22,31 +25,40 @@ import log as logging
 
 LOG = logging.getLogger(__name__)
 
-
 class Credentials(object):
 
-    def get_credentials(self):
+    def get_session(self):
-        dct = {}
+        dct = {
-        dct['username'] = self.rc_username
+            'username': self.rc_username,
-        dct['password'] = self.rc_password
+            'password': self.rc_password,
-        dct['auth_url'] = self.rc_auth_url
+            'auth_url': self.rc_auth_url
-        if self.rc_identity_api_version == 3:
+        }
-            dct['project_name'] = self.rc_project_name
+        auth = None
-            dct['project_domain_name'] = self.rc_project_domain_name
-            dct['user_domain_name'] = self.rc_user_domain_name
-        else:
-            dct['tenant_name'] = self.rc_tenant_name
-        return dct
 
-    def _init_with_openrc_(self, openrc_contents):
+        if self.rc_identity_api_version == 3:
+            dct.update({
+                'project_name': self.rc_project_name,
+                'project_domain_name': self.rc_project_domain_name,
+                'user_domain_name': self.rc_user_domain_name
+            })
+            auth = v3.Password(**dct)
+        else:
+            dct.update({
+                'tenant_name': self.rc_tenant_name
+            })
+            auth = v2.Password(**dct)
+        return session.Session(auth=auth, verify=self.rc_cacert)
+
+    def __parse_openrc(self, file):
         export_re = re.compile('export OS_([A-Z_]*)="?(.*)')
-        for line in openrc_contents.splitlines():
+        for line in file:
             line = line.strip()
-            mstr = export_re.match(line.strip())
+            mstr = export_re.match(line)
             if mstr:
                 # get rif of posible trailing double quote
                 # the first one was removed by the re
-                name, value = mstr.group(1), mstr.group(2)
+                name = mstr.group(1)
+                value = mstr.group(2)
                 if value.endswith('"'):
                     value = value[:-1]
                 # get rid of password assignment
@@ -61,7 +73,7 @@ class Credentials(object):
                     self.rc_identity_api_version = int(value)
 
                 # now match against wanted variable names
-                if name == 'USERNAME':
+                elif name == 'USERNAME':
                     self.rc_username = value
                 elif name == 'AUTH_URL':
                     self.rc_auth_url = value
@@ -73,39 +85,39 @@ class Credentials(object):
                     self.rc_region_name = value
                 elif name == "PASSWORD":
                     self.rc_password = value
+                elif name == "USER_DOMAIN_NAME":
+                    self.rc_user_domain_name = value
                 elif name == "PROJECT_NAME":
                     self.rc_project_name = value
                 elif name == "PROJECT_DOMAIN_NAME":
                     self.rc_project_domain_name = value
-                elif name == "USER_DOMAIN_NAME":
-                    self.rc_user_domain_name = value
 
+    #
     # Read a openrc file and take care of the password
     # The 2 args are passed from the command line and can be None
-    def __init__(self, openrc_file=None, openrc_contents=None, pwd=None, no_env=False):
+    #
+    def __init__(self, openrc_file, pwd, no_env):
         self.rc_password = None
         self.rc_username = None
         self.rc_tenant_name = None
         self.rc_auth_url = None
-        self.rc_cacert = False
+        self.rc_cacert = None
         self.rc_region_name = None
-        self.rc_project_name = None
-        self.rc_project_domain_name = None
         self.rc_user_domain_name = None
-        self.rc_identity_api_version = 2
+        self.rc_project_domain_name = None
-        self.openrc_contents = openrc_contents
+        self.rc_project_name = None
+        self.rc_identity_api_version = '2'
         success = True
 
         if openrc_file:
-            if os.path.exists(openrc_file):
+            if isinstance(openrc_file, str):
-                self.openrc_contents = open(openrc_file).read()
+                if os.path.exists(openrc_file):
+                    self.__parse_openrc(open(openrc_file))
+                else:
+                    LOG.error('Error: rc file does not exist %s', openrc_file)
+                    success = False
             else:
-                LOG.error("rc file does not exist %s" % openrc_file)
+                self.__parse_openrc(openrc_file)
-                success = False
-                return
-
-        if self.openrc_contents:
-            self._init_with_openrc_(self.openrc_contents)
         elif not no_env:
             # no openrc file passed - we assume the variables have been
             # sourced by the calling shell
@@ -122,10 +134,8 @@ class Credentials(object):
                     self.rc_username = os.environ['OS_USERNAME']
                     self.rc_auth_url = os.environ['OS_AUTH_URL']
                     self.rc_tenant_name = os.environ['OS_TENANT_NAME']
-
                 if 'OS_REGION_NAME' in os.environ:
                     self.rc_region_name = os.environ['OS_REGION_NAME']
-
             elif self.rc_identity_api_version == 3:
                 for varname in ['OS_USERNAME', 'OS_AUTH_URL', 'OS_PROJECT_NAME',
                                 'OS_PROJECT_DOMAIN_NAME', 'OS_USER_DOMAIN_NAME']:
@@ -136,12 +146,12 @@ class Credentials(object):
                     self.rc_username = os.environ['OS_USERNAME']
                     self.rc_auth_url = os.environ['OS_AUTH_URL']
                     self.rc_project_name = os.environ['OS_PROJECT_NAME']
-                    self.rc_project_domain_name = os.environ['OS_PROJECT_DOMAIN_NAME']
+                    self.rc_project_domain_id = os.environ['OS_PROJECT_DOMAIN_NAME']
-                    self.rc_user_domain_name = os.environ['OS_USER_DOMAIN_NAME']
+                    self.rc_user_domain_id = os.environ['OS_USER_DOMAIN_NAME']
-
             if 'OS_CACERT' in os.environ:
                 self.rc_cacert = os.environ['OS_CACERT']
 
+
         # always override with CLI argument if provided
         if pwd:
             self.rc_password = pwd

kloudbuster/force_cleanup.py

@@ -56,10 +56,7 @@ import time
 
 # openstack python clients
 import cinderclient
-from keystoneclient.auth.identity import v2 as keystone_v2
-from keystoneclient.auth.identity import v3 as keystone_v3
 from keystoneclient import client as keystoneclient
-from keystoneclient import session
 import neutronclient
 from novaclient.exceptions import NotFound
 from tabulate import tabulate
@@ -472,11 +469,7 @@ class KbCleaners(object):
     def __init__(self, creds_obj, resources, dryrun):
         self.cleaners = []
         creds = creds_obj.get_credentials()
-        if creds_obj.rc_identity_api_version == 3:
+        sess = creds.get_session()
-            auth = keystone_v3.Password(**creds)
-        else:
-            auth = keystone_v2.Password(**creds)
-        sess = session.Session(auth=auth, verify=creds_obj.rc_cacert)
         for cleaner_type in [StorageCleaner, ComputeCleaner, NetworkCleaner, KeystoneCleaner]:
             self.cleaners.append(cleaner_type(sess, resources, dryrun))
 

kloudbuster/kloudbuster.py

@@ -25,8 +25,8 @@ import webbrowser
 
 import base_compute
 import base_network
-from glanceclient import client as glanceclient
 import glanceclient.exc as glance_exception
+from glanceclient.v2 import client as glanceclient
 from kb_config import KBConfig
 from kb_res_logger import KBResLogger
 from kb_runner_base import KBException
@@ -35,10 +35,9 @@ from kb_runner_multicast import KBRunner_Multicast
 from kb_runner_storage import KBRunner_Storage
 from kb_scheduler import KBScheduler
 import kb_vm_agent
-from keystoneclient.auth.identity import v2 as keystone_v2
+
-from keystoneclient.auth.identity import v3 as keystone_v3
 from keystoneclient import client as keystoneclient
-from keystoneclient import session
+
 import log as logging
 from novaclient import client as novaclient
 from oslo_config import cfg
@@ -56,19 +55,6 @@ __version__ = pbr.version.VersionInfo('kloudbuster').version_string_with_vcs()
 class KBVMCreationException(Exception):
     pass
 
-def create_auth_session(creds_obj):
-    """
-    Return the authenticated session
-    """
-    creds = creds_obj.get_credentials()
-    if creds_obj.rc_identity_api_version == 3:
-        auth = keystone_v3.Password(**creds)
-    else:
-        auth = keystone_v2.Password(**creds)
-    sess = session.Session(auth=auth, verify=creds_obj.rc_cacert)
-
-    return sess
-
 class Kloud(object):
     def __init__(self, scale_cfg, cred, reusing_tenants,
                  testing_side=False, storage_mode=False, multicast_mode=False):
@@ -78,7 +64,7 @@ class Kloud(object):
         self.reusing_tenants = reusing_tenants
         self.storage_mode = storage_mode
         self.multicast_mode = multicast_mode
-        self.osclient_session = create_auth_session(cred)
+        self.osclient_session = cred.get_session()
         self.flavor_to_use = None
         self.vm_up_count = 0
         self.res_logger = KBResLogger()
@@ -92,10 +78,8 @@ class Kloud(object):
         self.placement_az = scale_cfg['availability_zone'] \
             if scale_cfg['availability_zone'] else None
         self.exc_info = None
-
         self.keystone = keystoneclient.Client(session=self.osclient_session,
                                               endpoint_type='publicURL')
-
         LOG.info("Creating kloud: " + self.prefix)
         if self.placement_az:
             LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az))
@@ -299,7 +283,7 @@ class KloudBuster(object):
 
     def get_hypervisor_list(self, cred):
         ret_list = []
-        sess = create_auth_session(cred)
+        sess = cred.get_session()
         nova_client = novaclient('2', endpoint_type='publicURL',
                                  http_log_debug=True, session=sess)
         for hypervisor in nova_client.hypervisors.list():
@@ -310,7 +294,7 @@ class KloudBuster(object):
 
     def get_az_list(self, cred):
         ret_list = []
-        sess = create_auth_session(cred)
+        sess = cred.get_session()
         nova_client = novaclient('2', endpoint_type='publicURL',
                                  http_log_debug=True, session=sess)
         for az in nova_client.availability_zones.list():
@@ -323,14 +307,14 @@ class KloudBuster(object):
 
     def check_and_upload_images(self, retry_count=150):
         retry = 0
-        creds_list = [create_auth_session(self.server_cred),
+        creds_list = [self.server_cred.get_session(),
-                      create_auth_session(self.client_cred)]
+                      self.client_cred.get_session()]
         creds_dict = dict(zip(['Server kloud', 'Client kloud'], creds_list))
         img_name_dict = dict(zip(['Server kloud', 'Client kloud'],
                                  [self.server_cfg.image_name, self.client_cfg.image_name]))
 
         for kloud, sess in creds_dict.items():
-            glance_client = glanceclient.Client('1', session=sess)
+            glance_client = glanceclient.Client('2', session=sess)
             try:
                 # Search for the image
                 img = glance_client.images.list(filters={'name': img_name_dict[kloud]}).next()

requirements.txt

@@ -2,19 +2,29 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-pbr>=1.3
-Babel>=1.3
-
-futures>=3.0.5
-python-openstackclient>=2.2.0
-python-neutronclient>=4.0.0
-attrdict>=2.0.0
-hdrhistogram>=0.3.1
-oslo.log>=1.0.0
-pecan>=0.9.0
-redis>=2.10.3
-tabulate>=0.7.3
 pytz>=2016.4
+pbr>=3.0.1
+Babel>=2.3.4
+
+futures>=3.1.1
+python-cinderclient>=2.0.1
+python-glanceclient>=2.6.0
+python-openstackclient>=3.11.0
+python-neutronclient>=6.2.0
+# migrate security group API to neutron client before moving to nova client 8.0.0
+python-novaclient<8.0.0
+python-keystoneclient>=3.10.0
+attrdict>=2.0.0
+hdrhistogram>=0.5.2
+# ipaddress is required to get TLS working
+# otherwise certificates with numeric IP addresses in the ServerAltName field will fail
+ipaddress>= 1.0.16
+oslo.config>=4.1.1
+oslo.log>=3.26.1
+pecan>=1.2.1
+redis>=2.10.5
+tabulate>=0.7.7
+pyyaml>=3.12
 
 # Workaround for pip install failed on RHEL/CentOS
 functools32>=3.2.3