From d06a86a8fda2af1790629d1d554e69362eb863cb Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 13:39:57 +0000 Subject: [PATCH 1/5] sync tests/charmhelpers --- .../contrib/openstack/amulet/deployment.py | 40 +++++++++++++++++++ .../contrib/openstack/amulet/utils.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 722bc645..f9304d45 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import re import six from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( @@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment): for service, config in six.iteritems(configs): self.d.configure(service, config) + def _auto_wait_for_status(self, message=None, exclude_services=None, + timeout=1800): + """Wait for all units to have a specific extended status, except + for any defined as excluded. Unless specified via message, any + status containing any case of 'ready' will be considered a match. + + Examples of message usage: + + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) + + Wait for all units to reach this status (exact match): + message = 'Unit is ready' + + Wait for all units to reach any one of these (exact match): + message = re.compile('Unit is ready|OK|Ready') + + Wait for at least one unit to reach this status (exact match): + message = {'ready'} + + See Amulet's sentry.wait_for_messages() for message usage detail. + https://github.com/juju/amulet/blob/master/amulet/sentry.py + + :param message: Expected status match + :param exclude_services: List of juju service names to ignore + :param timeout: Maximum time in seconds to wait for status match + :returns: None. Raises if timeout is hit. + """ + + if not message: + message = re.compile('.*ready.*', re.IGNORECASE) + + if not exclude_services: + exclude_services = [] + + services = list(set(self.d.services.keys()) - set(exclude_services)) + service_messages = {service: message for service in services} + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + def _get_openstack_release(self): """Get openstack release. diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index b1397419..2b3087ea 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -752,7 +752,7 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('SSL is enabled @{}:{} ' '({})'.format(host, port, unit_name)) return True - elif not port and not conf_ssl: + elif not conf_ssl: self.log.debug('SSL not enabled @{}:{} ' '({})'.format(host, port, unit_name)) return False From 46bd4988eea68e871b8eebbd1fab614228d3ce8a Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 13:40:43 +0000 Subject: [PATCH 2/5] sync hooks/charmhelpers --- .../contrib/openstack/amulet/deployment.py | 40 ++++++++++++++ .../contrib/openstack/amulet/utils.py | 2 +- .../charmhelpers/contrib/openstack/context.py | 53 ++++++++++++++----- .../charmhelpers/contrib/openstack/neutron.py | 20 +++++-- .../contrib/openstack/templates/ceph.conf | 6 +++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/core/hookenv.py | 1 - hooks/charmhelpers/core/host.py | 13 ++++- 8 files changed, 117 insertions(+), 19 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 722bc645..f9304d45 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import re import six from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( @@ -114,6 +115,45 @@ class OpenStackAmuletDeployment(AmuletDeployment): for service, config in six.iteritems(configs): self.d.configure(service, config) + def _auto_wait_for_status(self, message=None, exclude_services=None, + timeout=1800): + """Wait for all units to have a specific extended status, except + for any defined as excluded. Unless specified via message, any + status containing any case of 'ready' will be considered a match. + + Examples of message usage: + + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) + + Wait for all units to reach this status (exact match): + message = 'Unit is ready' + + Wait for all units to reach any one of these (exact match): + message = re.compile('Unit is ready|OK|Ready') + + Wait for at least one unit to reach this status (exact match): + message = {'ready'} + + See Amulet's sentry.wait_for_messages() for message usage detail. + https://github.com/juju/amulet/blob/master/amulet/sentry.py + + :param message: Expected status match + :param exclude_services: List of juju service names to ignore + :param timeout: Maximum time in seconds to wait for status match + :returns: None. Raises if timeout is hit. + """ + + if not message: + message = re.compile('.*ready.*', re.IGNORECASE) + + if not exclude_services: + exclude_services = [] + + services = list(set(self.d.services.keys()) - set(exclude_services)) + service_messages = {service: message for service in services} + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) + def _get_openstack_release(self): """Get openstack release. diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index b1397419..2b3087ea 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -752,7 +752,7 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('SSL is enabled @{}:{} ' '({})'.format(host, port, unit_name)) return True - elif not port and not conf_ssl: + elif not conf_ssl: self.log.debug('SSL not enabled @{}:{} ' '({})'.format(host, port, unit_name)) return False diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 1248d49f..48216338 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import glob import json import os import re @@ -951,6 +952,19 @@ class NeutronContext(OSContextGenerator): 'config': config} return ovs_ctxt + def midonet_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + midonet_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + mido_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'midonet', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': midonet_config} + + return mido_ctxt + def __call__(self): if self.network_manager not in ['quantum', 'neutron']: return {} @@ -972,6 +986,8 @@ class NeutronContext(OSContextGenerator): ctxt.update(self.nuage_ctxt()) elif self.plugin == 'plumgrid': ctxt.update(self.pg_ctxt()) + elif self.plugin == 'midonet': + ctxt.update(self.midonet_ctxt()) alchemy_flags = config('neutron-alchemy-flags') if alchemy_flags: @@ -1104,7 +1120,7 @@ class SubordinateConfigContext(OSContextGenerator): ctxt = { ... other context ... - 'subordinate_config': { + 'subordinate_configuration': { 'DEFAULT': { 'key1': 'value1', }, @@ -1145,22 +1161,23 @@ class SubordinateConfigContext(OSContextGenerator): try: sub_config = json.loads(sub_config) except: - log('Could not parse JSON from subordinate_config ' - 'setting from %s' % rid, level=ERROR) + log('Could not parse JSON from ' + 'subordinate_configuration setting from %s' + % rid, level=ERROR) continue for service in self.services: if service not in sub_config: - log('Found subordinate_config on %s but it contained' - 'nothing for %s service' % (rid, service), - level=INFO) + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s service' + % (rid, service), level=INFO) continue sub_config = sub_config[service] if self.config_file not in sub_config: - log('Found subordinate_config on %s but it contained' - 'nothing for %s' % (rid, self.config_file), - level=INFO) + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s' + % (rid, self.config_file), level=INFO) continue sub_config = sub_config[self.config_file] @@ -1363,7 +1380,7 @@ class DataPortContext(NeutronPortContext): normalized.update({port: port for port in resolved if port in ports}) if resolved: - return {bridge: normalized[port] for port, bridge in + return {normalized[port]: bridge for port, bridge in six.iteritems(portmap) if port in normalized.keys()} return None @@ -1374,12 +1391,22 @@ class PhyNICMTUContext(DataPortContext): def __call__(self): ctxt = {} mappings = super(PhyNICMTUContext, self).__call__() - if mappings and mappings.values(): - ports = mappings.values() + if mappings and mappings.keys(): + ports = sorted(mappings.keys()) napi_settings = NeutronAPIContext()() mtu = napi_settings.get('network_device_mtu') + all_ports = set() + # If any of ports is a vlan device, its underlying device must have + # mtu applied first. + for port in ports: + for lport in glob.glob("/sys/class/net/%s/lower_*" % port): + lport = os.path.basename(lport) + all_ports.add(lport.split('_')[1]) + + all_ports = list(all_ports) + all_ports.extend(ports) if mtu: - ctxt["devs"] = '\\n'.join(ports) + ctxt["devs"] = '\\n'.join(all_ports) ctxt['mtu'] = mtu return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 55b2037f..c54d63c7 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -209,6 +209,20 @@ def neutron_plugins(): 'server_packages': ['neutron-server', 'neutron-plugin-plumgrid'], 'server_services': ['neutron-server'] + }, + 'midonet': { + 'config': '/etc/neutron/plugins/midonet/midonet.ini', + 'driver': 'midonet.neutron.plugin.MidonetPluginV2', + 'contexts': [ + context.SharedDBContext(user=config('neutron-database-user'), + database=config('neutron-database'), + relation_prefix='neutron', + ssl_dir=NEUTRON_CONF_DIR)], + 'services': [], + 'packages': [[headers_package()] + determine_dkms_package()], + 'server_packages': ['neutron-server', + 'python-neutron-plugin-midonet'], + 'server_services': ['neutron-server'] } } if release >= 'icehouse': @@ -310,10 +324,10 @@ def parse_bridge_mappings(mappings): def parse_data_port_mappings(mappings, default_bridge='br-data'): """Parse data port mappings. - Mappings must be a space-delimited list of port:bridge mappings. + Mappings must be a space-delimited list of bridge:port. - Returns dict of the form {port:bridge} where port may be an mac address or - interface name. + Returns dict of the form {port:bridge} where ports may be mac addresses or + interface names. """ # NOTE(dosaboy): we use rvalue for key to allow multiple values to be diff --git a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf index b99851cc..33ceee25 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/ceph.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/ceph.conf @@ -13,3 +13,9 @@ log to syslog = {{ use_syslog }} err to syslog = {{ use_syslog }} clog to syslog = {{ use_syslog }} +[client] +{% if rbd_client_cache_settings -%} +{% for key, value in rbd_client_cache_settings.iteritems() -%} +{{ key }} = {{ value }} +{% endfor -%} +{%- endif %} \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index eefcf08b..cf4e5547 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -121,6 +121,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.2.2', 'kilo'), ('2.3.0', 'liberty'), ('2.4.0', 'liberty'), + ('2.5.0', 'liberty'), ]) # >= Liberty version->codename mapping diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 40d0a8f0..c2bee134 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -795,7 +795,6 @@ def status_set(workload_state, message): raise log_message = 'status-set failed: {} {}'.format(workload_state, message) - # XXX Fix this log(log_message, level='INFO') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index cb3c527e..d6d2d312 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -566,7 +566,14 @@ def chdir(d): os.chdir(cur) -def chownr(path, owner, group, follow_links=True): +def chownr(path, owner, group, follow_links=True, chowntopdir=False): + """ + Recursively change user and group ownership of files and directories + in given path. Doesn't chown path itself by default, only its children. + + :param bool follow_links: Also Chown links if True + :param bool chowntopdir: Also chown path itself if True + """ uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid if follow_links: @@ -574,6 +581,10 @@ def chownr(path, owner, group, follow_links=True): else: chown = os.lchown + if chowntopdir: + broken_symlink = os.path.lexists(path) and not os.path.exists(path) + if not broken_symlink: + chown(path, uid, gid) for root, dirs, files in os.walk(path): for name in dirs + files: full = os.path.join(root, name) From e252f0b711db7108b53d32e55cae650dcd5dc5a1 Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 13:41:59 +0000 Subject: [PATCH 3/5] enable liberty amulet tests --- tests/020-basic-trusty-liberty | 0 tests/021-basic-wily-liberty | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/020-basic-trusty-liberty mode change 100644 => 100755 tests/021-basic-wily-liberty diff --git a/tests/020-basic-trusty-liberty b/tests/020-basic-trusty-liberty old mode 100644 new mode 100755 diff --git a/tests/021-basic-wily-liberty b/tests/021-basic-wily-liberty old mode 100644 new mode 100755 From 902921bea6a193ee54596c699c0673cec490a69c Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 14:27:22 +0000 Subject: [PATCH 4/5] wait for workload status before testing; add service and relations to satisfy workload status ready state. --- tests/basic_deployment.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 290de08c..f28df92b 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -1,11 +1,5 @@ -#!/usr/bin/python -""" -Basic neutron-api functional test. -""" - import amulet import os -import time import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( @@ -35,6 +29,11 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): self._add_relations() self._configure_services() self._deploy() + + u.log.info('Waiting on extended status checks...') + exclude_services = ['mysql'] + self._auto_wait_for_status(exclude_services=exclude_services) + self._initialize_tests() def _add_services(self): @@ -48,6 +47,7 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): other_services = [{'name': 'mysql'}, {'name': 'rabbitmq-server'}, {'name': 'keystone'}, + {'name': 'glance'}, # to satisfy workload status {'name': 'neutron-openvswitch'}, {'name': 'nova-cloud-controller'}, {'name': 'neutron-gateway'}, @@ -68,6 +68,19 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): 'nova-compute:neutron-plugin': 'neutron-openvswitch:' 'neutron-plugin', 'nova-cloud-controller:shared-db': 'mysql:shared-db', + 'neutron-gateway:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:amqp': 'rabbitmq-server:amqp', + 'nova-compute:amqp': 'rabbitmq-server:amqp', + 'neutron-openvswitch:amqp': 'rabbitmq-server:amqp', + 'nova-cloud-controller:identity-service': 'keystone:' + 'identity-service', + 'nova-cloud-controller:cloud-compute': 'nova-compute:' + 'cloud-compute', + 'glance:identity-service': 'keystone:identity-service', + 'glance:shared-db': 'mysql:shared-db', + 'glance:amqp': 'rabbitmq-server:amqp', + 'nova-compute:image-service': 'glance:image-service', + 'nova-cloud-controller:image-service': 'glance:image-service', } # NOTE(beisner): relate this separately due to the resulting @@ -158,8 +171,6 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): self._get_openstack_release())) u.log.debug('openstack release str: {}'.format( self._get_openstack_release_string())) - # Let things settle a bit before moving forward - time.sleep(30) def test_100_services(self): """Verify the expected services are running on the corresponding From b7ec1fa5fe0e179dbaefd0a16f1271b5bc882a5e Mon Sep 17 00:00:00 2001 From: Ryan Beisner Date: Fri, 16 Oct 2015 16:13:56 +0000 Subject: [PATCH 5/5] fix relation check, remove varying unit data check --- tests/basic_deployment.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index f28df92b..2a3472d8 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -237,13 +237,6 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment): 'password': u.not_null } - if self._get_openstack_release() == self.precise_icehouse: - # Precise - expected['allowed_units'] = 'nova-cloud-controller/0 neutron-api/0' - else: - # Not Precise - expected['allowed_units'] = 'neutron-api/0' - ret = u.validate_relation_data(unit, relation, expected) if ret: message = u.relation_error('mysql shared-db', ret)