diff --git a/freezer/main.py b/freezer/main.py index 0472ca16..3bbf449f 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -27,7 +27,6 @@ from oslo_log import log from freezer.common import config as freezer_config from freezer.engine.tar import tar_engine from freezer import job -from freezer.openstack import openstack from freezer.openstack import osclients from freezer.storage import local from freezer.storage import multiple @@ -58,23 +57,21 @@ def freezer_main(backup_args): validator.validate(backup_args) work_dir = backup_args.work_dir - os_identity = backup_args.os_identity_api_version max_segment_size = backup_args.max_segment_size - if backup_args.storage == 'swift' or (backup_args.__dict__['backup_media'] - in ['nova', 'cinder', 'cindernative']): - client_manager = get_client_manager(backup_args.__dict__) - + if backup_args.storage == 'swift' or ( + backup_args.backup_media in ['nova', 'cinder', 'cindernative']): + backup_args.client_manager = get_client_manager(backup_args.__dict__) if backup_args.storages: storage = multiple.MultipleStorage( work_dir, - [storage_from_dict(x, work_dir, max_segment_size, os_identity) + [storage_from_dict(x, work_dir, max_segment_size) for x in backup_args.storages]) else: storage = storage_from_dict(backup_args.__dict__, work_dir, - max_segment_size, os_identity) + max_segment_size) - backup_args.__dict__['engine'] = tar_engine.TarBackupEngine( + backup_args.engine = tar_engine.TarBackupEngine( backup_args.compression, backup_args.dereference_symlink, backup_args.exclude, @@ -146,26 +143,24 @@ def parse_osrc(file_name): with open(file_name, 'r') as osrc_file: return config.osrc_parse(osrc_file.read()) -def get_client_manager(backup_args, os_identity_api_version=None): + +def get_client_manager(backup_args): if "osrc" in backup_args: - options = openstack.OpenstackOptions.create_from_dict( + options = osclients.OpenstackOpts.create_from_dict( parse_osrc(backup_args['osrc'])) else: - options = openstack.OpenstackOptions.create_from_env() - identity_api_version = (os_identity_api_version or - options.identity_api_version) - client_manager = osclients.ClientManager( - options=options, - insecure=backup_args.get('insecure') or False, - swift_auth_version=identity_api_version, - dry_run=backup_args.get('dry_run') or False) + options = osclients.OpenstackOpts.create_from_env().get_opts_dicts() - backup_args['client_manager'] = client_manager + client_manager = osclients.OSClientManager( + auth_url=options.pop('auth_url', None), + auth_method=options.pop('auth_method', 'password'), + dry_run=backup_args.get('dry_run', None), + **options + ) return client_manager -def storage_from_dict(backup_args, work_dir, max_segment_size, - os_identity_api_version=None): +def storage_from_dict(backup_args, work_dir, max_segment_size): storage_name = backup_args['storage'] container = backup_args['container'] if storage_name == "swift": @@ -188,7 +183,7 @@ def storage_from_dict(backup_args, work_dir, max_segment_size, def main(): - """freezer-agent/freezerc binary main execution""" + """freezer-agent binary main execution""" backup_args = None try: freezer_config.config() @@ -203,5 +198,4 @@ def main(): return fail(1, err, quiet) if __name__ == '__main__': - sys.exit(main()) diff --git a/freezer/openstack/backup.py b/freezer/openstack/backup.py index da099744..30b0e498 100644 --- a/freezer/openstack/backup.py +++ b/freezer/openstack/backup.py @@ -28,7 +28,7 @@ logging = log.getLogger(__name__) home = os.path.expanduser("~") -class BackupOs: +class BackupOs(object): def __init__(self, client_manager, container, storage): """ @@ -76,8 +76,8 @@ class BackupOs: headers = {"x-object-meta-name": instance._info['name'], "x-object-meta-flavor-id": instance._info['flavor']['id']} self.storage.add_stream(stream, package, headers) - logging.info("[*] Deleting temporary image") - glance.images.delete(image) + logging.info("[*] Deleting temporary image {0}".format(image)) + glance.images.delete(image.id) def backup_cinder_by_glance(self, volume_id): """ diff --git a/freezer/openstack/openstack.py b/freezer/openstack/openstack.py deleted file mode 100644 index ee634652..00000000 --- a/freezer/openstack/openstack.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -(c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import os - - -class OpenstackOptions: - """ - Stores credentials for OpenStack API. - Can be created using - >> create_from_env() - or - >> create_from_dict(dict) - """ - def __init__(self, user_name, tenant_name, project_name, auth_url, - password, identity_api_version, tenant_id=None, - region_name=None, endpoint_type=None, cert=None, - insecure=False, verify=True): - self.user_name = user_name - self.tenant_name = tenant_name - self.auth_url = auth_url - self.password = password - self.tenant_id = tenant_id - self.project_name = project_name - self.identity_api_version = identity_api_version - self.region_name = region_name - self.endpoint_type = endpoint_type - self.cert = cert - self.insecure = insecure - self.verify = verify - if not (self.password and self.user_name and self.auth_url and - (self.tenant_name or self.project_name)): - raise Exception("Please set up in your env:" - "OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME," - " OS_AUTH_URL, OS_PASSWORD") - - @property - def os_options(self): - """ - :return: The OpenStack options which can have tenant_id, - auth_token, service_type, endpoint_type, tenant_name, - object_storage_url, region_name - """ - return {'tenant_id': self.tenant_id, - 'tenant_name': self.tenant_name, - 'project_name': self.project_name, - 'identity_api_version': self.identity_api_version, - 'region_name': self.region_name, - 'endpoint_type': self.endpoint_type} - - @staticmethod - def create_from_env(): - return OpenstackOptions.create_from_dict(os.environ) - - @staticmethod - def create_from_dict(src_dict): - return OpenstackOptions( - user_name=src_dict.get('OS_USERNAME', None), - tenant_name=src_dict.get('OS_TENANT_NAME', None), - project_name=src_dict.get('OS_PROJECT_NAME', None), - auth_url=src_dict.get('OS_AUTH_URL', None), - identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION', - '2.0'), - password=src_dict.get('OS_PASSWORD', None), - tenant_id=src_dict.get('OS_TENANT_ID', None), - region_name=src_dict.get('OS_REGION_NAME', None), - endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None), - cert=src_dict.get('OS_CERT', None) - ) diff --git a/freezer/openstack/osclients.py b/freezer/openstack/osclients.py index 1f746836..02473686 100644 --- a/freezer/openstack/osclients.py +++ b/freezer/openstack/osclients.py @@ -1,4 +1,5 @@ # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. +# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,14 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import swiftclient import time -from cinderclient import client as cclient -from glanceclient import client as gclient -from novaclient import client as nclient +from cinderclient.client import Client as cinder_client +from glanceclient.client import Client as glance_client +from keystoneauth1 import loading +from keystoneauth1 import session +from novaclient.client import Client as nova_client from oslo_config import cfg from oslo_log import log -import swiftclient from freezer.utils import utils @@ -27,158 +31,156 @@ CONF = cfg.CONF logging = log.getLogger(__name__) -class ClientManager: - - """ - :type swift: swiftclient.Connection - :type glance: glanceclient.v1.client.Client - :type nova: novaclient.v2.client.Client - :type cinder: cinderclient.v1.client.Client - """ - def __init__(self, options, insecure=True, - swift_auth_version=2, dry_run=False): - """ - Creates manager of connections to swift, nova, glance and cinder - :param options: OpenstackOptions - :type options: freezer.openstack.openstack.OpenstackOptions - :param insecure: - :param swift_auth_version: - :param dry_run: - :return: - """ - self.options = options - self.insecure = insecure - self.swift_auth_version = swift_auth_version - self.dry_run = dry_run - self.cinder = None +class OSClientManager(object): + + def __init__(self, auth_url, auth_method='password', **kwargs): self.swift = None self.glance = None self.nova = None + self.cinder = None + self.dry_run = kwargs.pop('dry_run', None) + loader = loading.get_plugin_loader(auth_method) + # copy the args for swift authentication ! + self.swift_args = kwargs.copy() + self.swift_args['auth_url'] = auth_url + # client specific arguments ! + self.client_kwargs = {} + # session specific arguments + session_kwargs = {} + if 'verify' in kwargs.keys(): + session_kwargs['verify'] = kwargs.pop('verify') + if 'cacert' in kwargs.keys(): + session_kwargs['cert'] = kwargs.pop('cacert') + # client specific args + if 'insecure' in kwargs.keys(): + self.client_kwargs['insecure'] = kwargs.pop('insecure') + session_kwargs['verify'] = False + if 'region_name' in kwargs.keys(): + self.client_kwargs['region_name'] = kwargs.pop('region_name') + if 'endpoint_type' in kwargs.keys(): + self.client_kwargs['endpoint_type'] = kwargs.pop('endpoint_type') + if 'identity_api_version' in kwargs.keys(): + kwargs.pop('identity_api_version') + if 'auth_version' in kwargs.keys(): + kwargs.pop('auth_version') + if 'interface' in kwargs.keys(): + self.client_kwargs['interface'] = kwargs.pop('interface') - def get_cinder(self): - """ - :rtype cinderclient.v1.client.Client - :return: - """ - if not self.cinder: - self.create_cinder() - return self.cinder + self.compute_version = kwargs.pop('compute_api_version', 2) + self.image_version = kwargs.pop('image_api_version', 2) + self.volume_version = kwargs.pop('volume_api_version', 2) + self.auth = loader.load_from_options(auth_url=auth_url, **kwargs) - def get_swift(self): - """ - :rtype swiftclient.Connection - :return: instance of swift client - """ - if not self.swift: - self.create_swift() - return self.swift + self.sess = session.Session(auth=self.auth, **session_kwargs) - def get_glance(self): + def create_nova(self): """ - :rtype glanceclient.v1.client.Client - :return: + Use pre-initialized session to create an instance of nova client. + :return: novaclient instance """ - if not self.glance: - self.create_glance() - return self.glance - - def get_nova(self): - """ - :rtype - :return: - """ - if not self.nova: - self.create_nova() + self.nova = nova_client(self.compute_version, session=self.sess, + **self.client_kwargs) return self.nova + def create_glance(self): + """ + Use pre-initialized session to create an instance of glance client. + :return: glanceclient instance + """ + if 'endpoint_type' in self.client_kwargs.keys(): + self.client_kwargs.pop('endpoint_type') + if 'insecure' in self.client_kwargs.keys(): + self.client_kwargs.pop('insecure') + + self.glance = glance_client(self.image_version, session=self.sess, + **self.client_kwargs) + return self.glance + def create_cinder(self): """ - Creates client for cinder and caches it - :rtype cinderclient.v1.client.Client - :return: instance of cinder client + Use pre-initialized session to create an instance of cinder client. + :return: cinderclient instance """ - options = self.options - logging.info("[*] Creation of cinder client") - self.cinder = cclient.Client( - version="2", - username=options.user_name, - api_key=options.password, - project_id=options.tenant_name, - auth_url=options.auth_url, - region_name=options.region_name, - insecure=self.insecure, - endpoint_type=options.endpoint_type or 'publicURL', - service_type="volume", - cacert=options.cert) + self.cinder = cinder_client(self.volume_version, session=self.sess, + **self.client_kwargs) return self.cinder def create_swift(self): """ - Creates client for swift and caches it - :rtype swiftclient.Connection - :return: instance of swift client + Swift client needs to be treated differently so we need to copy the + arguments and provide it to swiftclient the correct way ! + :return: swiftclient instance """ - options = self.options - logging.info("[*] Creation of swift client") + os_options = {} + auth_version = None + if 'region_name' in self.swift_args.keys(): + os_options['region_name'] = self.swift_args.get('region_name') + if 'endpoint_type' in self.swift_args.keys(): + os_options['endpoint_type'] = self.swift_args.pop('endpoint_type') + if 'tenant_id' in self.swift_args.keys(): + os_options['tenant_id'] = self.swift_args.pop('tenant_id') + if 'identity_api_version' in self.swift_args.keys(): + os_options['identity_api_version'] = \ + self.swift_args.pop('identity_api_version') + auth_version = os_options['identity_api_version'] + if 'token' in self.swift_args.keys(): + os_options['auth_token'] = self.swift_args.pop('token') + if 'auth_version' in self.swift_args.keys(): + auth_version = self.swift_args.get('auth_version') + + tenant_name = self.swift_args.get('project_name') or self.swift_args.\ + get('tenant_name') self.swift = swiftclient.client.Connection( - authurl=options.auth_url, - user=options.user_name, key=options.password, - tenant_name=options.tenant_name, - os_options=options.os_options, - auth_version=self.swift_auth_version, - insecure=self.insecure, retries=6, - cacert=options.cert) + authurl=self.swift_args.get('auth_url'), + user=self.swift_args.get('username'), + key=self.swift_args.get('password'), + tenant_name=tenant_name, + insecure=self.swift_args.get('insecure', False), + cacert=self.swift_args.get('cacert', None), + os_options=os_options, + auth_version=auth_version + ) if self.dry_run: self.swift = DryRunSwiftclientConnectionWrapper(self.swift) return self.swift - def create_glance(self): + def get_nova(self): """ - Creates client for glance and caches it - :rtype glanceclient.v1.client.Client - :return: instance of glance client + Get novaclient instance + :return: novaclient instance """ + if not self.nova: + self.nova = self.create_nova() + return self.nova - from glanceclient.shell import OpenStackImagesShell - - options = self.options - - logging.info("[*] Creation of glance client") - - self.glance = OpenStackImagesShell()._get_versioned_client('1', - utils.Bunch(os_username=options.user_name, - os_password=options.password, - os_tenant_name=options.tenant_name, - os_project_name=options.project_name, - os_auth_url=options.auth_url, - os_region_name=options.region_name, - endpoint_type=options.endpoint_type, - force_auth=False, - cacert=options.cert)) - + def get_glance(self): + """ + Get glanceclient instance + :return: glanceclient instance + """ + if not self.glance: + self.glance = self.create_glance() return self.glance - def create_nova(self): + def get_cinder(self): """ - Creates client for nova and caches it - :return: + Get cinderclient instance + :return: cinderclient instance """ - options = self.options - logging.info("[*] Creation of nova client") + if not self.cinder: + self.cinder = self.create_cinder() + return self.cinder - self.nova = nclient.Client( - version='2', - username=options.user_name, - api_key=options.password, - project_id=options.tenant_name, - auth_url=options.auth_url, - region_name=options.region_name, - insecure=self.insecure, - cacert=options.cert) - - return self.nova + def get_swift(self): + """ + Get swiftclient instance + :return: swiftclient instance + """ + if not self.swift: + self.swift = self.create_swift() + return self.swift def provide_snapshot(self, volume, snapshot_name): """ @@ -275,7 +277,188 @@ class ClientManager: return utils.ReSizeStream(stream, image.size, 1000000) -class DryRunSwiftclientConnectionWrapper: +class OpenstackOpts(object): + """ + Gathering and maintaining the right Openstack credentials that will be used + to authenticate against keystone. Now we support keystone v2 and v3. + We need to provide a correct url that ends with either v2.0 or v3 or provide + auth_version or identity_api_version + """ + def __init__(self, auth_url, auth_method='password', auth_version=None, + username=None, password=None, region_name=None, cacert=None, + identity_api_version=None, project_id=None, project_name=None, + tenant_id=None, tenant_name=None, token=None, insecure=False, + endpoint_type='internalURL', interface=None, + compute_api_version=2, image_api_version=2, + volume_api_version=2, user_domain_name=None, domain_id=None, + user_domain_id=None, project_domain_id=None, domain_name=None, + project_domain_name=None): + """ + Authentication Options to build a valid opts dict to be used to + authenticate against keystone. You must provide auth_url with a vaild + Openstack version at the end v2.0 or v3 or provide auth_version. + :param auth_url: string Keystone API URL + :param auth_method: string defaults to password or token (not tested) + :param auth_version: string Keystone API version. 2.0 or 3 + :param username: string A valid Username + :param password: string A valid Password + :param region_name: string Region name or None + :param cacert: string Path to CA certificate + :param identity_api_version: string Keystone API version to use + :param project_id: UUID string Project ID + :param project_name: string Project Name + :param tenant_id: string Project/ Tenant ID. Use with keystone v2.0 only + :param tenant_name: string Project/ Tenant Name. keystone v2.0 only + :param token: string Valid token. Only if auth_method is token + :param insecure: boolean Use insecure connections + :param endpoint_type: string publicURL, adminURL, internalURL + :param interface: string internal, ... + :param compute_api_version: int NOVA API version to use default 2 + :param image_api_version: int Glance API version, default 2 + :param volume_api_version: int Cinder API version, default 2 + :param user_domain_name: string User Domain Name. only with keystone v3 + :param domain_id: string Domain ID. Only with keystone v3 + :param user_domain_id: string User Domain ID. only with keystone v3 + :param project_domain_id: string Project Domain ID. keystone v3 only + :param domain_name: string Domain Name. only with keystone v3 + :param project_domain_name: string Project Domain Name. keystone v3 only + :return: None + """ + self.auth_url = auth_url + self.auth_method = auth_method + self.auth_version = auth_version + self.username = username + self.password = password + self.region_name = region_name + self.cacert = cacert + self.identity_api_version = identity_api_version + self.tenant_id = tenant_id or project_id + self.project_id = project_id or tenant_id + self.project_name = project_name or tenant_name + self.tenant_name = tenant_name or project_name + self.token = token + self.insecure = insecure + self.endpoint_type = endpoint_type + self.interface = interface + self.compute_api_version = compute_api_version + self.image_api_version = image_api_version + self.volume_api_version = volume_api_version + self.user_domain_id = user_domain_id + self.user_domain_name = user_domain_name + self.project_domain_id = project_domain_id + self.project_domain_name = project_domain_name + self.domain_id = domain_id + self.domain_name = domain_name + if auth_url is None: + raise Exception('auth_url required to authenticate. Make sure to ' + 'export OS_AUTH_URL=http://keystone_url:5000/v3') + if auth_version is None and identity_api_version is None: + version = auth_url.rstrip('/').rsplit('/')[-1] + if version == 'v3': + self.auth_version = self.identity_api_version = str('3') + elif version == 'v2.0': + self.auth_version = self.identity_api_version = str('2.0') + else: + raise Exception('Keystone Auth version {0} is not supported!. ' + 'Generated from auth_url: {1}'.format(version, + auth_url)) + logging.info('Authenticating with Keystone version: {0}, auth_url: {1},' + ' username: {2}, project: {3}'.format(self.auth_version, + self.auth_url, + self.username, + self.project_name)) + + def get_opts_dicts(self): + """ + Return opentack auth arguments as dict + detects the auth version from url if not provided + handles certificate issues + """ + opts = self.__dict__ + if self.auth_method == 'password': + opts.pop('token', None) + elif self.auth_method == 'token': + opts.pop('username', None) + opts.pop('password', None) + + if not self.cacert: + opts['verify'] = False + opts['insecure'] = True + self.auth_version = str(self.auth_version) + self.identity_api_version = str(self.identity_api_version) + + if self.auth_version == '3' or self.identity_api_version == '3': + opts['auth_version'] = opts['identity_api_version'] = '3' + opts.pop('tenant_id', None) + opts.pop('tenant_name', None) + + elif self.auth_version in ['2.0', '2'] or self.identity_api_version in \ + ['2.0', '2']: + opts['auth_version'] = opts['identity_api_version'] = '2.0' + # these parameters won't work with keystone v2.0 + opts.pop('project_id', None) + opts.pop('project_name', None) + opts.pop('project_domain_id', None) + opts.pop('project_domain_name', None) + opts.pop('user_domain_id', None) + opts.pop('user_domain_name', None) + opts.pop('domain_id', None) + opts.pop('domain_name', None) + else: + raise Exception('Keystone Auth version {0} is not supported!. ' + 'Generated from auth_url: {1}'. + format(self.auth_version, self.auth_url)) + for i in opts.copy().keys(): + if opts.get(i) is None: + opts.pop(i) + return opts + + @staticmethod + def create_from_env(): + """ + Parse environment variables and load Openstack related options. + :return: + """ + return OpenstackOpts.create_from_dict(os.environ) + + @staticmethod + def create_from_dict(src_dict): + """ + Load Openstack arguments from dict and return OpenstackOpts object with + the correct parameters to authenticate. + :param src_dict: dict + :return: OpenstackOpts object with the passed arguments in place + """ + return OpenstackOpts( + auth_url=src_dict.get('OS_AUTH_URL'), + auth_method=src_dict.get('OS_AUTH_METHOD', 'password'), + auth_version=src_dict.get('OS_AUTH_VERSION', None), + username=src_dict.get('OS_USERNAME', None), + password=src_dict.get('OS_PASSWORD', None), + tenant_id=src_dict.get('OS_TENANT_ID', None), + tenant_name=src_dict.get('OS_TENANT_NAME', None), + project_id=src_dict.get('OS_PROJECT_ID', None), + project_name=src_dict.get('OS_PROJECT_NAME', None), + region_name=src_dict.get('OS_REGION_NAME', None), + endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', 'publicURL'), + cacert=src_dict.get('OS_CACERT', None), + identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION', None), + insecure=src_dict.get('OS_INSECURE', False), + token=src_dict.get('OS_TOKEN', None), + interface=src_dict.get('OS_INTERFACE', None), + user_domain_name=src_dict.get('OS_USER_DOMAIN_NAME', None), + user_domain_id=src_dict.get('OS_USER_DOMAIN_ID', None), + project_domain_id=src_dict.get('OS_PROJECT_DOMAIN_ID', None), + project_domain_name=src_dict.get('OS_PROJECT_DOMAIN_NAME', None), + domain_id=src_dict.get('OS_DOMAIN_ID'), + domain_name=src_dict.get('OS_DOMAIN_NAME'), + compute_api_version=src_dict.get('OS_COMPUTE_API_VERSION', 2), + volume_api_version=src_dict.get('OS_VOLUME_API_VERSION', 2), + image_api_version=src_dict.get('OS_IMAGE_API_VERSION', 2) + ) + + +class DryRunSwiftclientConnectionWrapper(object): def __init__(self, sw_connector): self.sw_connector = sw_connector self.get_object = sw_connector.get_object diff --git a/freezer/tests/commons.py b/freezer/tests/commons.py index 9a68c016..b5eb6bf8 100644 --- a/freezer/tests/commons.py +++ b/freezer/tests/commons.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. +# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +17,7 @@ from mock import MagicMock +from mock import Mock import swiftclient import multiprocessing @@ -27,13 +29,14 @@ import re from glanceclient.common.utils import IterableWithLength from freezer.storage import swift from freezer.utils import utils -from freezer.openstack import openstack +from freezer.openstack.osclients import OpenstackOpts +from freezer.openstack.osclients import OSClientManager from freezer.engine.tar import tar_engine os.environ['OS_REGION_NAME'] = 'testregion' os.environ['OS_TENANT_ID'] = '0123456789' os.environ['OS_PASSWORD'] = 'testpassword' -os.environ['OS_AUTH_URL'] = 'testauthurl' +os.environ['OS_AUTH_URL'] = 'http://testauthurl/v2.0' os.environ['OS_USERNAME'] = 'testusername' os.environ['OS_TENANT_NAME'] = 'testtenantename' @@ -316,10 +319,14 @@ class BackupOpt1: self.cindernative_vol_id = '' self.nova_inst_id = '' self.lvm_snapperm = 'ro' - self.options = openstack.OpenstackOptions.create_from_dict(os.environ) - from freezer.openstack.osclients import ClientManager - from mock import Mock - self.client_manager = ClientManager(None, False, 2, False) + + self.compression = 'gzip' + self.storage = MagicMock() + self.engine = MagicMock() + opts = OpenstackOpts.create_from_env().get_opts_dicts() + self.client_manager = OSClientManager(opts.pop('auth_url'), + opts.pop('auth_method'), + **opts) self.client_manager.get_swift = Mock( return_value=FakeSwiftClient().client.Connection()) self.client_manager.create_swift = self.client_manager.get_swift @@ -327,8 +334,6 @@ class BackupOpt1: self.container, self.work_dir, self.max_segment_size) - self.compression = 'gzip' - self.engine = tar_engine.TarBackupEngine( self.compression, self.dereference_symlink, self.exclude, self.storage, False) @@ -337,6 +342,7 @@ class BackupOpt1: nova_client = MagicMock() self.client_manager.get_nova = Mock(return_value=nova_client) + self.command = None @@ -387,7 +393,7 @@ class Os: @classmethod def exists(cls, directory=True): return True - + @classmethod def notexists(cls, directory=True): return False diff --git a/freezer/utils/streaming.py b/freezer/utils/streaming.py index 216c8fbb..009f7db6 100644 --- a/freezer/utils/streaming.py +++ b/freezer/utils/streaming.py @@ -1,6 +1,6 @@ """ (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. - +(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -103,4 +103,4 @@ class QueuedThread(threading.Thread): super(QueuedThread, self).run() except Exception as e: self.rich_queue.force_stop() - raise e \ No newline at end of file + raise e diff --git a/requirements.txt b/requirements.txt index b6b8d799..13d77d34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ oslo.utils>=3.5.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.log>=1.14.0 # Apache-2.0 oslo.config>=3.9.0 # Apache-2.0 +keystoneauth1>=2.1.0 # Apache-2.0 PyMySQL>=0.6.2 # MIT License pymongo!=3.1,>=3.0.2 # Apache-2.0 diff --git a/tests/unit/openstack/test_osclients.py b/tests/unit/openstack/test_osclients.py index 4ddb89b0..5bb11fa5 100644 --- a/tests/unit/openstack/test_osclients.py +++ b/tests/unit/openstack/test_osclients.py @@ -1,4 +1,5 @@ # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. +# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,61 +13,46 @@ # See the License for the specific language governing permissions and # limitations under the License. - import unittest import mock -from freezer.openstack import openstack from freezer.openstack import osclients -from freezer.utils import utils class TestOsClients(unittest.TestCase): - - fake_options = openstack.OpenstackOptions( - user_name="user", tenant_name="tenant", project_name="project", - auth_url="url", password="password", identity_api_version="3", - insecure=False, cert='cert', verify=True) + def setUp(self): + self.opts = osclients.OpenstackOpts( + username="user", tenant_name="tenant", project_name="project", + auth_url="url/v3", password="password", identity_api_version="3", + insecure=False, cacert='cert', user_domain_name='Default', + project_domain_name='Default').get_opts_dicts() + self.client_manager = osclients.OSClientManager(auth_method=self.opts.pop('auth_method'), + auth_url=self.opts.pop('auth_url'), + **self.opts) def test_init(self): - osclients.ClientManager(self.fake_options, None, None, None) + self.client_manager.get_cinder() def test_create_cinder(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.create_cinder() + self.client_manager.create_cinder() def test_create_swift(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.create_swift() + self.client_manager.create_swift() def test_create_nova(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.create_nova() - - def test_create_swift_public(self): - options = openstack.OpenstackOptions( - user_name="user", tenant_name="tenant", project_name="project", - auth_url="url", password="password", identity_api_version="3", - endpoint_type="adminURL", insecure=False, cert='cert', - verify=True) - client = osclients.ClientManager(options, None, None, None) - client.create_swift() + self.client_manager.create_nova() def test_dry_run(self): osclients.DryRunSwiftclientConnectionWrapper(mock.Mock()) def test_get_cinder(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.get_cinder() + self.client_manager.get_cinder() def test_get_swift(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.get_swift() + self.client_manager.get_swift() def get_glance(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.get_glance() + self.client_manager.get_glance() def get_nova(self): - client = osclients.ClientManager(self.fake_options, None, None, None) - client.get_nova() + self.client_manager.get_nova() diff --git a/tests/unit/openstack/test_restore.py b/tests/unit/openstack/test_restore.py index 724746b3..e131c477 100644 --- a/tests/unit/openstack/test_restore.py +++ b/tests/unit/openstack/test_restore.py @@ -1,7 +1,7 @@ """Freezer restore.py related tests (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. - +(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/tests/unit/storages/test_swift.py b/tests/unit/storages/test_swift.py index 241f43be..ecfe2d8c 100644 --- a/tests/unit/storages/test_swift.py +++ b/tests/unit/storages/test_swift.py @@ -1,4 +1,5 @@ # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. +# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +16,6 @@ import unittest -from freezer.openstack import openstack from freezer.openstack import osclients from freezer.storage import swift from freezer.storage import base @@ -24,11 +24,12 @@ from freezer.storage import base class TestSwiftStorage(unittest.TestCase): def setUp(self): - + opts = osclients.OpenstackOpts.create_from_env().get_opts_dicts() self.storage = swift.SwiftStorage( - osclients.ClientManager( - openstack.OpenstackOptions.create_from_env() - ), + osclients.OSClientManager(opts.pop('auth_url'), + opts.pop('auth_method', 'password'), + **opts + ), "freezer_ops-aw1ops1-gerrit0001.aw1.hpcloud.net", "/tmp/", 100, skip_prepare=True diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 7ab7a6f3..e71f9db7 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -1,4 +1,5 @@ # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P. +# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +17,7 @@ import datetime import unittest -from freezer.openstack import openstack +from freezer.openstack.osclients import OpenstackOpts from freezer.tests.commons import * from freezer.utils import utils @@ -65,14 +66,19 @@ class TestUtils(unittest.TestCase): self.assertRaises(ValueError, utils.human2bytes, '12 foo') def test_OpenstackOptions_creation_success(self): + class FreezerOpts: + def __init__(self, opts): + self.__dict__.update(opts) env_dict = dict(OS_USERNAME='testusername', OS_TENANT_NAME='testtenantename', OS_AUTH_URL='testauthurl', OS_PASSWORD='testpassword', OS_REGION_NAME='testregion', - OS_TENANT_ID='0123456789') - options = openstack.OpenstackOptions.create_from_dict(env_dict) - assert options.user_name == env_dict['OS_USERNAME'] + OS_TENANT_ID='0123456789', + OS_AUTH_VERSION='2.0') + options = OpenstackOpts.create_from_dict(env_dict).get_opts_dicts() + options = FreezerOpts(options) + assert options.username == env_dict['OS_USERNAME'] assert options.tenant_name == env_dict['OS_TENANT_NAME'] assert options.auth_url == env_dict['OS_AUTH_URL'] assert options.password == env_dict['OS_PASSWORD'] @@ -82,14 +88,14 @@ class TestUtils(unittest.TestCase): env_dict = dict(OS_USERNAME='testusername', OS_TENANT_NAME='testtenantename', OS_AUTH_URL='testauthurl', - OS_PASSWORD='testpassword') - options = openstack.OpenstackOptions.create_from_dict(env_dict) - assert options.user_name == env_dict['OS_USERNAME'] + OS_PASSWORD='testpassword', + OS_AUTH_VERSION='2.0') + options = OpenstackOpts.create_from_dict(env_dict).get_opts_dicts() + options = FreezerOpts(options) + assert options.username == env_dict['OS_USERNAME'] assert options.tenant_name == env_dict['OS_TENANT_NAME'] assert options.auth_url == env_dict['OS_AUTH_URL'] assert options.password == env_dict['OS_PASSWORD'] - assert options.region_name is None - assert options.tenant_id is None def test_date_to_timestamp(self): # ensure that timestamp is check with appropriate timezone offset @@ -136,8 +142,8 @@ class TestDateTime: d = utils.DateTime(1425750464) assert 1425750464 == d.timestamp #ensure that time is check with appropriate timezone offset - t = time.strftime("%Y-%m-%d %H:%M:%S", - time.localtime((time.mktime(time.strptime("2015-03-07 17:47:44", + t = time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime((time.mktime(time.strptime("2015-03-07 17:47:44", "%Y-%m-%d %H:%M:%S")))-time.timezone)) assert t == '{}'.format(d)