diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index a871ce37..b13277bb 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import ( network_get_primary_address, unit_get, WARNING, + NoNetworkBinding, ) from charmhelpers.core.host import ( @@ -109,7 +110,12 @@ def get_address_in_network(network, fallback=None, fatal=False): _validate_cidr(network) network = netaddr.IPNetwork(network) for iface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(iface) + try: + addresses = netifaces.ifaddresses(iface) + except ValueError: + # If an instance was deleted between + # netifaces.interfaces() run and now, its interfaces are gone + continue if network.version == 4 and netifaces.AF_INET in addresses: for addr in addresses[netifaces.AF_INET]: cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], @@ -578,6 +584,9 @@ def get_relation_ip(interface, cidr_network=None): except NotImplementedError: # If network-get is not available address = get_host_ip(unit_get('private-address')) + except NoNetworkBinding: + log("No network binding for {}".format(interface), WARNING) + address = get_host_ip(unit_get('private-address')) if config('prefer-ipv6'): # Currently IPv6 has priority, eventually we want IPv6 to just be diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index 87f364d1..d93cff3c 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils): return 'endpoint not found' def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): + public_port, expected, expected_num_eps=3): """Validate keystone v3 endpoint data. Validate the v3 endpoint data which has changed from v2. The @@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return 'unexpected endpoint data - {}'.format(ret) - if len(found) != 3: + if len(found) != expected_num_eps: return 'Unexpected number of endpoints found' def validate_svc_catalog_endpoint_data(self, expected, actual): diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 7ada2760..36cf32fc 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -617,7 +617,9 @@ class HAProxyContext(OSContextGenerator): """ interfaces = ['cluster'] - def __init__(self, singlenode_mode=False): + def __init__(self, singlenode_mode=False, + address_types=ADDRESS_TYPES): + self.address_types = address_types self.singlenode_mode = singlenode_mode def __call__(self): @@ -631,7 +633,7 @@ class HAProxyContext(OSContextGenerator): # NOTE(jamespage): build out map of configured network endpoints # and associated backends - for addr_type in ADDRESS_TYPES: + for addr_type in self.address_types: cfg_opt = 'os-{}-network'.format(addr_type) # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather # than 'internal' @@ -1635,18 +1637,84 @@ class InternalEndpointContext(OSContextGenerator): endpoints by default so this allows admins to optionally use internal endpoints. """ - def __init__(self, ost_rel_check_pkg_name): - self.ost_rel_check_pkg_name = ost_rel_check_pkg_name + def __call__(self): + return {'use_internal_endpoints': config('use-internal-endpoints')} + + +class VolumeAPIContext(InternalEndpointContext): + """Volume API context. + + This context provides information regarding the volume endpoint to use + when communicating between services. It determines which version of the + API is appropriate for use. + + This value will be determined in the resulting context dictionary + returned from calling the VolumeAPIContext object. Information provided + by this context is as follows: + + volume_api_version: the volume api version to use, currently + 'v2' or 'v3' + volume_catalog_info: the information to use for a cinder client + configuration that consumes API endpoints from the keystone + catalog. This is defined as the type:name:endpoint_type string. + """ + # FIXME(wolsen) This implementation is based on the provider being able + # to specify the package version to check but does not guarantee that the + # volume service api version selected is available. In practice, it is + # quite likely the volume service *is* providing the v3 volume service. + # This should be resolved when the service-discovery spec is implemented. + def __init__(self, pkg): + """ + Creates a new VolumeAPIContext for use in determining which version + of the Volume API should be used for communication. A package codename + should be supplied for determining the currently installed OpenStack + version. + + :param pkg: the package codename to use in order to determine the + component version (e.g. nova-common). See + charmhelpers.contrib.openstack.utils.PACKAGE_CODENAMES for more. + """ + super(VolumeAPIContext, self).__init__() + self._ctxt = None + if not pkg: + raise ValueError('package name must be provided in order to ' + 'determine current OpenStack version.') + self.pkg = pkg + + @property + def ctxt(self): + if self._ctxt is not None: + return self._ctxt + self._ctxt = self._determine_ctxt() + return self._ctxt + + def _determine_ctxt(self): + """Determines the Volume API endpoint information. + + Determines the appropriate version of the API that should be used + as well as the catalog_info string that would be supplied. Returns + a dict containing the volume_api_version and the volume_catalog_info. + """ + rel = os_release(self.pkg, base='icehouse') + version = '2' + if CompareOpenStackReleases(rel) >= 'pike': + version = '3' + + service_type = 'volumev{version}'.format(version=version) + service_name = 'cinderv{version}'.format(version=version) + endpoint_type = 'publicURL' + if config('use-internal-endpoints'): + endpoint_type = 'internalURL' + catalog_info = '{type}:{name}:{endpoint}'.format( + type=service_type, name=service_name, endpoint=endpoint_type) + + return { + 'volume_api_version': version, + 'volume_catalog_info': catalog_info, + } def __call__(self): - ctxt = {'use_internal_endpoints': config('use-internal-endpoints')} - rel = os_release(self.ost_rel_check_pkg_name, base='icehouse') - if CompareOpenStackReleases(rel) >= 'pike': - ctxt['volume_api_version'] = '3' - else: - ctxt['volume_api_version'] = '2' - - return ctxt + return self.ctxt class AppArmorContext(OSContextGenerator): @@ -1784,3 +1852,30 @@ class MemcacheContext(OSContextGenerator): ctxt['memcache_server_formatted'], ctxt['memcache_port']) return ctxt + + +class EnsureDirContext(OSContextGenerator): + ''' + Serves as a generic context to create a directory as a side-effect. + + Useful for software that supports drop-in files (.d) in conjunction + with config option-based templates. Examples include: + * OpenStack oslo.policy drop-in files; + * systemd drop-in config files; + * other software that supports overriding defaults with .d files + + Another use-case is when a subordinate generates a configuration for + primary to render in a separate directory. + + Some software requires a user to create a target directory to be + scanned for drop-in files with a specific format. This is why this + context is needed to do that before rendering a template. + ''' + + def __init__(self, dirname): + '''Used merely to ensure that a given directory exists.''' + self.dirname = dirname + + def __call__(self): + mkdir(self.dirname) + return {} diff --git a/charmhelpers/contrib/openstack/templating.py b/charmhelpers/contrib/openstack/templating.py index 77490e4d..a623315d 100644 --- a/charmhelpers/contrib/openstack/templating.py +++ b/charmhelpers/contrib/openstack/templating.py @@ -93,7 +93,8 @@ class OSConfigTemplate(object): Associates a config file template with a list of context generators. Responsible for constructing a template context based on those generators. """ - def __init__(self, config_file, contexts): + + def __init__(self, config_file, contexts, config_template=None): self.config_file = config_file if hasattr(contexts, '__call__'): @@ -103,6 +104,8 @@ class OSConfigTemplate(object): self._complete_contexts = [] + self.config_template = config_template + def context(self): ctxt = {} for context in self.contexts: @@ -124,6 +127,11 @@ class OSConfigTemplate(object): self.context() return self._complete_contexts + @property + def is_string_template(self): + """:returns: Boolean if this instance is a template initialised with a string""" + return self.config_template is not None + class OSConfigRenderer(object): """ @@ -148,6 +156,10 @@ class OSConfigRenderer(object): contexts=[context.IdentityServiceContext()]) configs.register(config_file='/etc/haproxy/haproxy.conf', contexts=[context.HAProxyContext()]) + configs.register(config_file='/etc/keystone/policy.d/extra.cfg', + contexts=[context.ExtraPolicyContext() + context.KeystoneContext()], + config_template=hookenv.config('extra-policy')) # write out a single config configs.write('/etc/nova/nova.conf') # write out all registered configs @@ -218,14 +230,23 @@ class OSConfigRenderer(object): else: apt_install('python3-jinja2') - def register(self, config_file, contexts): + def register(self, config_file, contexts, config_template=None): """ Register a config file with a list of context generators to be called during rendering. + config_template can be used to load a template from a string instead of + using template loaders and template files. + :param config_file (str): a path where a config file will be rendered + :param contexts (list): a list of context dictionaries with kv pairs + :param config_template (str): an optional template string to use """ - self.templates[config_file] = OSConfigTemplate(config_file=config_file, - contexts=contexts) - log('Registered config file: %s' % config_file, level=INFO) + self.templates[config_file] = OSConfigTemplate( + config_file=config_file, + contexts=contexts, + config_template=config_template + ) + log('Registered config file: {}'.format(config_file), + level=INFO) def _get_tmpl_env(self): if not self._tmpl_env: @@ -235,32 +256,58 @@ class OSConfigRenderer(object): def _get_template(self, template): self._get_tmpl_env() template = self._tmpl_env.get_template(template) - log('Loaded template from %s' % template.filename, level=INFO) + log('Loaded template from {}'.format(template.filename), + level=INFO) + return template + + def _get_template_from_string(self, ostmpl): + ''' + Get a jinja2 template object from a string. + :param ostmpl: OSConfigTemplate to use as a data source. + ''' + self._get_tmpl_env() + template = self._tmpl_env.from_string(ostmpl.config_template) + log('Loaded a template from a string for {}'.format( + ostmpl.config_file), + level=INFO) return template def render(self, config_file): if config_file not in self.templates: - log('Config not registered: %s' % config_file, level=ERROR) + log('Config not registered: {}'.format(config_file), level=ERROR) raise OSConfigException - ctxt = self.templates[config_file].context() - _tmpl = os.path.basename(config_file) - try: - template = self._get_template(_tmpl) - except exceptions.TemplateNotFound: - # if no template is found with basename, try looking for it - # using a munged full path, eg: - # /etc/apache2/apache2.conf -> etc_apache2_apache2.conf - _tmpl = '_'.join(config_file.split('/')[1:]) + ostmpl = self.templates[config_file] + ctxt = ostmpl.context() + + if ostmpl.is_string_template: + template = self._get_template_from_string(ostmpl) + log('Rendering from a string template: ' + '{}'.format(config_file), + level=INFO) + else: + _tmpl = os.path.basename(config_file) try: template = self._get_template(_tmpl) - except exceptions.TemplateNotFound as e: - log('Could not load template from %s by %s or %s.' % - (self.templates_dir, os.path.basename(config_file), _tmpl), - level=ERROR) - raise e + except exceptions.TemplateNotFound: + # if no template is found with basename, try looking + # for it using a munged full path, eg: + # /etc/apache2/apache2.conf -> etc_apache2_apache2.conf + _tmpl = '_'.join(config_file.split('/')[1:]) + try: + template = self._get_template(_tmpl) + except exceptions.TemplateNotFound as e: + log('Could not load template from {} by {} or {}.' + ''.format( + self.templates_dir, + os.path.basename(config_file), + _tmpl + ), + level=ERROR) + raise e - log('Rendering from template: %s' % _tmpl, level=INFO) + log('Rendering from template: {}'.format(config_file), + level=INFO) return template.render(ctxt) def write(self, config_file): diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 211ae87d..7ed1cc4e 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -820,6 +820,10 @@ class Hooks(object): return wrapper +class NoNetworkBinding(Exception): + pass + + def charm_dir(): """Return the root directory of the current charm""" d = os.environ.get('JUJU_CHARM_DIR') @@ -1106,7 +1110,17 @@ def network_get_primary_address(binding): :raise: NotImplementedError if run on Juju < 2.0 ''' cmd = ['network-get', '--primary-address', binding] - return subprocess.check_output(cmd).decode('UTF-8').strip() + try: + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + except CalledProcessError as e: + if 'no network config found for binding' in e.output.decode('UTF-8'): + raise NoNetworkBinding("No network binding for {}" + .format(binding)) + else: + raise + return response @translate_exc(from_exc=OSError, to_exc=NotImplementedError) diff --git a/charmhelpers/core/templating.py b/charmhelpers/core/templating.py index 7b801a34..9014015c 100644 --- a/charmhelpers/core/templating.py +++ b/charmhelpers/core/templating.py @@ -20,7 +20,8 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): + perms=0o444, templates_dir=None, encoding='UTF-8', + template_loader=None, config_template=None): """ Render a template. @@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root', The context should be a dict containing the values to be replaced in the template. + config_template may be provided to render from a provided template instead + of loading from a file. + The `owner`, `group`, and `perms` options will be passed to `write_file`. If omitted, `templates_dir` defaults to the `templates` folder in the charm. @@ -65,14 +69,19 @@ def render(source, target, context, owner='root', group='root', if templates_dir is None: templates_dir = os.path.join(hookenv.charm_dir(), 'templates') template_env = Environment(loader=FileSystemLoader(templates_dir)) - try: - source = source - template = template_env.get_template(source) - except exceptions.TemplateNotFound as e: - hookenv.log('Could not load template %s from %s.' % - (source, templates_dir), - level=hookenv.ERROR) - raise e + + # load from a string if provided explicitly + if config_template is not None: + template = template_env.from_string(config_template) + else: + try: + source = source + template = template_env.get_template(source) + except exceptions.TemplateNotFound as e: + hookenv.log('Could not load template %s from %s.' % + (source, templates_dir), + level=hookenv.ERROR) + raise e content = template.render(context) if target is not None: target_dir = os.path.dirname(target) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index b8dae2cf..2f01247f 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -48,8 +48,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): """Deploy the entire test environment.""" super(KeystoneBasicDeployment, self).__init__(series, openstack, source, stable) - self.keystone_num_units = 3 - self.keystone_api_version = 2 + + self._initialize_deployment_differences() self._setup_test_object(snap_source) self._add_services() @@ -63,6 +63,11 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): self.d.sentry.wait() self._initialize_tests() + self._initialize_test_differences() + + def _initialize_deployment_differences(self): + self.keystone_num_units = 3 + self.keystone_api_version = 2 def _setup_test_object(self, snap_source): self.snap_source = snap_source @@ -181,22 +186,23 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def set_api_version(self, api_version): # Avoid costly settings if we are already at the correct api_version - if not self.api_change_required(api_version): - return True - u.log.debug('Setting preferred-api-version={}'.format(api_version)) - se_rels = [] - for i in range(0, self.keystone_num_units): - se_rels.append( - (self.keystone_sentries[i], 'cinder:identity-service'), - ) - # Make config change, wait for propagation - u.keystone_configure_api_version(se_rels, self, api_version) + if self.api_change_required(api_version): + u.log.debug('Setting preferred-api-version={}'.format(api_version)) + se_rels = [] + for i in range(0, self.keystone_num_units): + se_rels.append( + (self.keystone_sentries[i], 'cinder:identity-service'), + ) + # Make config change, wait for propagation + u.keystone_configure_api_version(se_rels, self, api_version) - # Success if we get here, get and store client. + # Store in self.keystone_client if api_version == 2: self.keystone_v2 = self.get_keystone_client(api_version=2) + self.keystone_client = self.keystone_v2 else: self.keystone_v3 = self.get_keystone_client(api_version=3) + self.keystone_client = self.keystone_v3 self.keystone_api_version = api_version def get_keystone_client(self, api_version=None, keystone_ip=None): @@ -221,41 +227,42 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): self.demo_tenant = 'demoTenant' self.demo_role = 'demoRole' self.demo_user = 'demoUser' - if not u.tenant_exists(self.keystone_v2, self.demo_tenant): - tenant = self.keystone_v2.tenants.create( + if not u.tenant_exists(self.keystone_client, self.demo_tenant): + tenant = self.keystone_client.tenants.create( tenant_name=self.demo_tenant, description='demo tenant', enabled=True) - self.keystone_v2.roles.create(name=self.demo_role) - self.keystone_v2.users.create(name=self.demo_user, - password='password', - tenant_id=tenant.id, - email='demo@demo.com') + self.keystone_client.roles.create(name=self.demo_role) + self.keystone_client.users.create(name=self.demo_user, + password='password', + tenant_id=tenant.id, + email='demo@demo.com') # Authenticate keystone demo self.keystone_demo = u.authenticate_keystone_user( - self.keystone_v2, user=self.demo_user, + self.keystone_client, user=self.demo_user, password='password', tenant=self.demo_tenant) def create_users_v3(self): # Create a demo tenant/role/user self.demo_project = 'demoProject' self.demo_user_v3 = 'demoUserV3' + self.demo_role = 'demoRoleV3' self.demo_domain_admin = 'demoDomainAdminV3' self.demo_domain = 'demoDomain' try: - domain = self.keystone_v3.domains.find(name=self.demo_domain) + domain = self.keystone_client.domains.find(name=self.demo_domain) except keystoneclient.exceptions.NotFound: - domain = self.keystone_v3.domains.create( + domain = self.keystone_client.domains.create( self.demo_domain, description='Demo Domain', enabled=True ) try: - self.keystone_v3.projects.find(name=self.demo_project) + self.keystone_client.projects.find(name=self.demo_project) except keystoneclient.exceptions.NotFound: - self.keystone_v3.projects.create( + self.keystone_client.projects.create( self.demo_project, domain, description='Demo Project', @@ -263,14 +270,14 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): ) try: - self.keystone_v3.roles.find(name=self.demo_role) + self.keystone_client.roles.find(name=self.demo_role) except keystoneclient.exceptions.NotFound: - self.keystone_v3.roles.create(name=self.demo_role) + self.keystone_client.roles.create(name=self.demo_role) - if not self.find_keystone_v3_user(self.keystone_v3, + if not self.find_keystone_v3_user(self.keystone_client, self.demo_user_v3, self.demo_domain): - self.keystone_v3.users.create( + self.keystone_client.users.create( self.demo_user_v3, domain=domain.id, project=self.demo_project, @@ -280,14 +287,14 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): enabled=True) try: - self.keystone_v3.roles.find(name='Admin') + self.keystone_client.roles.find(name='Admin') except keystoneclient.exceptions.NotFound: - self.keystone_v3.roles.create(name='Admin') + self.keystone_client.roles.create(name='Admin') - if not self.find_keystone_v3_user(self.keystone_v3, + if not self.find_keystone_v3_user(self.keystone_client, self.demo_domain_admin, self.demo_domain): - user = self.keystone_v3.users.create( + user = self.keystone_client.users.create( self.demo_domain_admin, domain=domain.id, project=self.demo_project, @@ -296,10 +303,10 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): description='Demo Admin', enabled=True) - role = self.keystone_v3.roles.find(name='Admin') - u.log.debug("self.keystone_v3.roles.grant('{}', user='{}', " + role = self.keystone_client.roles.find(name='Admin') + u.log.debug("self.keystone_client.roles.grant('{}', user='{}', " "domain='{}')".format(role.id, user.id, domain.id)) - self.keystone_v3.roles.grant( + self.keystone_client.roles.grant( role.id, user=user.id, domain=domain.id) @@ -319,6 +326,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): self.keystone_ip = self.keystone_sentries[0].relation( 'shared-db', 'percona-cluster:shared-db')['private-address'] + + def _initialize_test_differences(self): self.set_api_version(2) self.create_users_v2() @@ -369,7 +378,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_102_keystone_tenants(self): self.set_api_version(2) - self.validate_keystone_tenants(self.keystone_v2) + self.validate_keystone_tenants(self.keystone_client) def validate_keystone_roles(self, client): """Verify all existing roles.""" @@ -388,7 +397,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_104_keystone_roles(self): self.set_api_version(2) - self.validate_keystone_roles(self.keystone_v2) + self.validate_keystone_roles(self.keystone_client) def validate_keystone_users(self, client): """Verify all existing roles.""" @@ -444,7 +453,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_106_keystone_users(self): self.set_api_version(2) - self.validate_keystone_users(self.keystone_v2) + self.validate_keystone_users(self.keystone_client) def is_liberty_or_newer(self): # os_release = self._get_openstack_release_string() @@ -469,15 +478,15 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_112_keystone_list_resources(self): if self.is_mitaka_or_newer(): self.set_api_version(3) - self.validate_keystone_tenants(self.keystone_v3) - self.validate_keystone_roles(self.keystone_v3) - self.validate_keystone_users(self.keystone_v3) + self.validate_keystone_tenants(self.keystone_client) + self.validate_keystone_roles(self.keystone_client) + self.validate_keystone_users(self.keystone_client) def test_118_keystone_create_users(self): if self.is_mitaka_or_newer(): self.set_api_version(3) self.create_users_v3() - actual_user = self.find_keystone_v3_user(self.keystone_v3, + actual_user = self.find_keystone_v3_user(self.keystone_client, self.demo_user_v3, self.demo_domain) assert actual_user is not None @@ -498,7 +507,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): if self.is_mitaka_or_newer(): self.set_api_version(3) self.create_users_v3() - actual_domain = self.keystone_v3.domains.find( + actual_domain = self.keystone_client.domains.find( name=self.demo_domain ) expect = { @@ -597,7 +606,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_140_keystone_endpoint(self): """Verify the keystone endpoint data.""" u.log.debug('Checking keystone api endpoint data...') - endpoints = self.keystone_v2.endpoints.list() + self.set_api_version(2) + endpoints = self.keystone_client.endpoints.list() admin_port = '35357' internal_port = public_port = '5000' expected = { @@ -617,7 +627,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): def test_142_cinder_endpoint(self): """Verify the cinder endpoint data.""" u.log.debug('Checking cinder endpoint...') - endpoints = self.keystone_v2.endpoints.list() + endpoints = self.keystone_client.endpoints.list() admin_port = internal_port = public_port = '8776' expected = { 'id': u.not_null, @@ -947,11 +957,11 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): message="Unit is ready", timeout=timeout, include_only=['keystone']) - domain = self.keystone_v3.domains.find(name='admin_domain') - v3_admin_user = self.keystone_v3.users.list(domain=domain)[0] + domain = self.keystone_client.domains.find(name='admin_domain') + v3_admin_user = self.keystone_client.users.list(domain=domain)[0] u.log.debug(v3_admin_user) - self.keystone_v3.users.update(user=v3_admin_user, - password='wrongpass') + self.keystone_client.users.update(user=v3_admin_user, + password='wrongpass') u.log.debug('Removing keystone percona-cluster relation') self.d.unrelate('keystone:shared-db', 'percona-cluster:shared-db') self.d.sentry.wait(timeout=timeout) @@ -976,3 +986,179 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment): amulet.FAIL, msg="Admin user password not reset") u.log.debug('OK') + + +class KeystoneV3Deployment(KeystoneBasicDeployment): + """Amulet tests on a basic keystone deployment.""" + + def _initialize_deployment_differences(self): + self.keystone_num_units = 3 + self.keystone_api_version = 3 + + def _initialize_test_differences(self): + self.keystone_client = self.get_keystone_client(api_version=3) + self.create_users_v3() + + def api_change_required(self, api_version): + u.log.warn('This is a Keystone V3 only deployment.') + return False + + def set_api_version(self, api_version): + u.log.warn('This is a Keystone V3 only deployment. ' + 'Ignoring request for api version 2') + + def validate_keystone_tenants(self, client): + """Verify all existing tenants.""" + u.log.debug('Checking keystone tenants...') + expected = [ + {'name': 'services', + 'enabled': True, + 'description': 'Created by Juju', + 'id': u.not_null}, + {'name': 'demoProject', + 'enabled': True, + 'description': 'Demo Project', + 'id': u.not_null}, + {'name': 'admin', + 'enabled': True, + 'description': 'Created by Juju', + 'id': u.not_null} + ] + actual = client.projects.list() + + ret = u.validate_tenant_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def validate_keystone_roles(self, client): + """Verify all existing roles.""" + u.log.debug('Checking keystone roles...') + expected = [ + {'name': 'demoRoleV3', + 'id': u.not_null}, + {'name': 'Admin', + 'id': u.not_null} + ] + actual = client.roles.list() + + ret = u.validate_role_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def validate_keystone_users(self, client): + """Verify all existing roles.""" + u.log.debug('Checking keystone users...') + + if self._get_openstack_release() < self.xenial_pike: + cinder_user = 'cinder_cinderv2' + else: + cinder_user = 'cinderv2_cinderv3' + base = [ + {'name': 'demoUserV3', + 'enabled': True, + 'id': u.not_null, + 'email': 'demov3@demo.com'}, + {'name': 'admin', + 'enabled': True, + 'id': u.not_null, + 'email': 'juju@localhost'}, + {'name': cinder_user, + 'enabled': True, + 'id': u.not_null, + 'email': u'juju@localhost'} + ] + expected = [] + for user_info in base: + user_info['default_project_id'] = u.not_null + expected.append(user_info) + # Ensure list is scoped to the default domain + # when checking v3 users (v2->v3 upgrade check) + actual = client.users.list( + domain=client.domains.find(name=self.DEFAULT_DOMAIN).id + ) + actual += client.users.list( + domain=client.domains.find(name=self.demo_domain).id) + actual += client.users.list( + domain=client.domains.find(name='admin_domain').id) + ret = u.validate_user_data(expected, actual, + api_version=self.keystone_api_version) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_138_service_catalog(self): + """Verify that the service catalog endpoint data is valid.""" + u.log.debug('Checking keystone service catalog...') + expected = { + u'identity': [{u'id': u.not_null, + u'interface': u'admin', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}, + {u'id': u.not_null, + u'interface': u'public', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}, + {u'id': u.not_null, + u'interface': u'internal', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}], + + u'volumev2': [{u'id': u.not_null, + u'interface': u'admin', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}, + {u'id': u.not_null, + u'interface': u'public', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}, + {u'id': u.not_null, + u'interface': u'internal', + u'region': u'RegionOne', + u'region_id': u'RegionOne', + u'url': u.valid_url}]} + + actual = self.keystone_client.service_catalog.get_endpoints() + ret = u.validate_v3_svc_catalog_endpoint_data(expected, actual) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_140_keystone_endpoint(self): + """Verify the keystone endpoint data.""" + u.log.debug('Checking keystone api endpoint data...') + admin_port = '35357' + internal_port = public_port = '5000' + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'region_id': 'RegionOne', + 'interface': u.not_null, + 'url': u.valid_url, + 'service_id': u.not_null} + + endpoints = self.keystone_client.endpoints.list() + ret = u.validate_v3_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected) + if ret: + amulet.raise_status(amulet.FAIL, + msg='keystone endpoint: {}'.format(ret)) + + def test_142_cinder_endpoint(self): + """Verify the cinder endpoint data.""" + u.log.debug('Checking cinder endpoint...') + admin_port = internal_port = public_port = '8776' + expected = {'id': u.not_null, + 'region': 'RegionOne', + 'region_id': 'RegionOne', + 'interface': u.not_null, + 'url': u.valid_url, + 'service_id': u.not_null} + endpoints = self.keystone_client.endpoints.list() + ret = u.validate_v3_endpoint_data(endpoints, admin_port, internal_port, + public_port, expected, + expected_num_eps=6) + if ret: + amulet.raise_status(amulet.FAIL, + msg='cinder endpoint: {}'.format(ret)) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 87f364d1..d93cff3c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils): return 'endpoint not found' def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): + public_port, expected, expected_num_eps=3): """Validate keystone v3 endpoint data. Validate the v3 endpoint data which has changed from v2. The @@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return 'unexpected endpoint data - {}'.format(ret) - if len(found) != 3: + if len(found) != expected_num_eps: return 'Unexpected number of endpoints found' def validate_svc_catalog_endpoint_data(self, expected, actual): diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index 211ae87d..7ed1cc4e 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -820,6 +820,10 @@ class Hooks(object): return wrapper +class NoNetworkBinding(Exception): + pass + + def charm_dir(): """Return the root directory of the current charm""" d = os.environ.get('JUJU_CHARM_DIR') @@ -1106,7 +1110,17 @@ def network_get_primary_address(binding): :raise: NotImplementedError if run on Juju < 2.0 ''' cmd = ['network-get', '--primary-address', binding] - return subprocess.check_output(cmd).decode('UTF-8').strip() + try: + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + except CalledProcessError as e: + if 'no network config found for binding' in e.output.decode('UTF-8'): + raise NoNetworkBinding("No network binding for {}" + .format(binding)) + else: + raise + return response @translate_exc(from_exc=OSError, to_exc=NotImplementedError) diff --git a/tests/charmhelpers/core/templating.py b/tests/charmhelpers/core/templating.py index 7b801a34..9014015c 100644 --- a/tests/charmhelpers/core/templating.py +++ b/tests/charmhelpers/core/templating.py @@ -20,7 +20,8 @@ from charmhelpers.core import hookenv def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): + perms=0o444, templates_dir=None, encoding='UTF-8', + template_loader=None, config_template=None): """ Render a template. @@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root', The context should be a dict containing the values to be replaced in the template. + config_template may be provided to render from a provided template instead + of loading from a file. + The `owner`, `group`, and `perms` options will be passed to `write_file`. If omitted, `templates_dir` defaults to the `templates` folder in the charm. @@ -65,14 +69,19 @@ def render(source, target, context, owner='root', group='root', if templates_dir is None: templates_dir = os.path.join(hookenv.charm_dir(), 'templates') template_env = Environment(loader=FileSystemLoader(templates_dir)) - try: - source = source - template = template_env.get_template(source) - except exceptions.TemplateNotFound as e: - hookenv.log('Could not load template %s from %s.' % - (source, templates_dir), - level=hookenv.ERROR) - raise e + + # load from a string if provided explicitly + if config_template is not None: + template = template_env.from_string(config_template) + else: + try: + source = source + template = template_env.get_template(source) + except exceptions.TemplateNotFound as e: + hookenv.log('Could not load template %s from %s.' % + (source, templates_dir), + level=hookenv.ERROR) + raise e content = template.render(context) if target is not None: target_dir = os.path.dirname(target) diff --git a/tests/dev-basic-bionic-queens b/tests/dev-basic-bionic-queens index ae1dd5aa..df46adc8 100755 --- a/tests/dev-basic-bionic-queens +++ b/tests/dev-basic-bionic-queens @@ -16,8 +16,8 @@ """Amulet tests on a basic keystone deployment on bionic-queens.""" -from basic_deployment import KeystoneBasicDeployment +from basic_deployment import KeystoneV3Deployment if __name__ == '__main__': - deployment = KeystoneBasicDeployment(series='bionic') + deployment = KeystoneV3Deployment(series='bionic') deployment.run_tests() diff --git a/tests/dev-basic-xenial-queens b/tests/gate-basic-xenial-queens similarity index 73% rename from tests/dev-basic-xenial-queens rename to tests/gate-basic-xenial-queens index 22b825b6..1aa6bbc7 100755 --- a/tests/dev-basic-xenial-queens +++ b/tests/gate-basic-xenial-queens @@ -16,10 +16,10 @@ """Amulet tests on a basic keystone deployment on xenial-queens.""" -from basic_deployment import KeystoneBasicDeployment +from basic_deployment import KeystoneV3Deployment if __name__ == '__main__': - deployment = KeystoneBasicDeployment(series='xenial', - openstack='cloud:xenial-queens', - source='cloud:xenial-updates/queens') + deployment = KeystoneV3Deployment(series='xenial', + openstack='cloud:xenial-queens', + source='cloud:xenial-updates/queens') deployment.run_tests() diff --git a/tox.ini b/tox.ini index 6d44f4b9..dae53621 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ skipsdist = True setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 CHARM_DIR={envdir} - AMULET_SETUP_TIMEOUT=2700 + AMULET_SETUP_TIMEOUT=5400 install_command = pip install --allow-unverified python-apt {opts} {packages} commands = ostestr {posargs}