diff --git a/.gitignore b/.gitignore index 3a1765c..2feddaa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ payload/* bin .testrepository *.sw[nop] +*.pyc diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..801646b --- /dev/null +++ b/.testr.conf @@ -0,0 +1,8 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 998f00c..4efe799 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -456,3 +456,18 @@ def get_hostname(address, fqdn=True): return result else: return result.split('.')[0] + + +def port_has_listener(address, port): + """ + Returns True if the address:port is open and being listened to, + else False. + + @param address: an IP address or hostname + @param port: integer port + + Note calls 'zc' via a subprocess shell + """ + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not(bool(result)) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index dbc489a..d057ea6 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -237,14 +237,16 @@ def neutron_plugins(): plugins['midonet']['driver'] = ( 'neutron.plugins.midonet.plugin.MidonetPluginV2') if release >= 'liberty': - midonet_origin = config('midonet-origin') - if midonet_origin is not None and midonet_origin[4:5] == '1': - plugins['midonet']['driver'] = ( - 'midonet.neutron.plugin_v1.MidonetPluginV2') - plugins['midonet']['server_packages'].remove( - 'python-neutron-plugin-midonet') - plugins['midonet']['server_packages'].append( - 'python-networking-midonet') + plugins['midonet']['driver'] = ( + 'midonet.neutron.plugin_v1.MidonetPluginV2') + plugins['midonet']['server_packages'].remove( + 'python-neutron-plugin-midonet') + plugins['midonet']['server_packages'].append( + 'python-networking-midonet') + plugins['plumgrid']['driver'] = ( + 'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2') + plugins['plumgrid']['server_packages'].remove( + 'neutron-plugin-plumgrid') return plugins diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend index ce28fa3..6a92380 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf index ce28fa3..6a92380 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 8077712..80dd2e0 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,6 +23,7 @@ import json import os import sys import re +import itertools import six import tempfile @@ -60,6 +61,7 @@ from charmhelpers.contrib.storage.linux.lvm import ( from charmhelpers.contrib.network.ip import ( get_ipv6_addr, is_ipv6, + port_has_listener, ) from charmhelpers.contrib.python.packages import ( @@ -67,7 +69,7 @@ from charmhelpers.contrib.python.packages import ( pip_install, ) -from charmhelpers.core.host import lsb_release, mounts, umount +from charmhelpers.core.host import lsb_release, mounts, umount, service_running from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -860,13 +862,23 @@ def os_workload_status(configs, required_interfaces, charm_func=None): return wrap -def set_os_workload_status(configs, required_interfaces, charm_func=None): +def set_os_workload_status(configs, required_interfaces, charm_func=None, services=None, ports=None): """ Set workload status based on complete contexts. status-set missing or incomplete contexts and juju-log details of missing required data. charm_func is a charm specific function to run checking for charm specific requirements such as a VIP setting. + + This function also checks for whether the services defined are ACTUALLY + running and that the ports they advertise are open and being listened to. + + @param services - OPTIONAL: a [{'service': , 'ports': []] + The ports are optional. + If services is a [] then ports are ignored. + @param ports - OPTIONAL: an [] representing ports that shoudl be + open. + @returns None """ incomplete_rel_data = incomplete_relation_data(configs, required_interfaces) state = 'active' @@ -945,6 +957,65 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None): else: message = charm_message + # If the charm thinks the unit is active, check that the actual services + # really are active. + if services is not None and state == 'active': + # if we're passed the dict() then just grab the values as a list. + if isinstance(services, dict): + services = services.values() + # either extract the list of services from the dictionary, or if + # it is a simple string, use that. i.e. works with mixed lists. + _s = [] + for s in services: + if isinstance(s, dict) and 'service' in s: + _s.append(s['service']) + if isinstance(s, str): + _s.append(s) + services_running = [service_running(s) for s in _s] + if not all(services_running): + not_running = [s for s, running in zip(_s, services_running) + if not running] + message = ("Services not running that should be: {}" + .format(", ".join(not_running))) + state = 'blocked' + # also verify that the ports that should be open are open + # NB, that ServiceManager objects only OPTIONALLY have ports + port_map = OrderedDict([(s['service'], s['ports']) + for s in services if 'ports' in s]) + if state == 'active' and port_map: + all_ports = list(itertools.chain(*port_map.values())) + ports_open = [port_has_listener('0.0.0.0', p) + for p in all_ports] + if not all(ports_open): + not_opened = [p for p, opened in zip(all_ports, ports_open) + if not opened] + map_not_open = OrderedDict() + for service, ports in port_map.items(): + closed_ports = set(ports).intersection(not_opened) + if closed_ports: + map_not_open[service] = closed_ports + # find which service has missing ports. They are in service + # order which makes it a bit easier. + message = ( + "Services with ports not open that should be: {}" + .format( + ", ".join([ + "{}: [{}]".format( + service, + ", ".join([str(v) for v in ports])) + for service, ports in map_not_open.items()]))) + state = 'blocked' + + if ports is not None and state == 'active': + # and we can also check ports which we don't know the service for + ports_open = [port_has_listener('0.0.0.0', p) for p in ports] + if not all(ports_open): + message = ( + "Ports which should be open, but are not: {}" + .format(", ".join([str(p) for p, v in zip(ports, ports_open) + if not v]))) + state = 'blocked' + # Set to active if all requirements have been met if state == 'active': message = "Unit is ready" diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 60ae52b..fb1bee3 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -120,6 +120,7 @@ class PoolCreationError(Exception): """ A custom error to inform the caller that a pool creation failed. Provides an error message """ + def __init__(self, message): super(PoolCreationError, self).__init__(message) @@ -129,6 +130,7 @@ class Pool(object): An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). """ + def __init__(self, service, name): self.service = service self.name = name @@ -180,36 +182,41 @@ class Pool(object): :return: int. The number of pgs to use. """ validator(value=pool_size, valid_type=int) - osds = get_osds(self.service) - if not osds: + osd_list = get_osds(self.service) + if not osd_list: # NOTE(james-page): Default to 200 for older ceph versions # which don't support OSD query from cli return 200 + osd_list_length = len(osd_list) # Calculate based on Ceph best practices - if osds < 5: + if osd_list_length < 5: return 128 - elif 5 < osds < 10: + elif 5 < osd_list_length < 10: return 512 - elif 10 < osds < 50: + elif 10 < osd_list_length < 50: return 4096 else: - estimate = (osds * 100) / pool_size + estimate = (osd_list_length * 100) / pool_size # Return the next nearest power of 2 index = bisect.bisect_right(powers_of_two, estimate) return powers_of_two[index] class ReplicatedPool(Pool): - def __init__(self, service, name, replicas=2): + def __init__(self, service, name, pg_num=None, replicas=2): super(ReplicatedPool, self).__init__(service=service, name=name) self.replicas = replicas + if pg_num is None: + self.pg_num = self.get_pgs(self.replicas) + else: + self.pg_num = pg_num def create(self): if not pool_exists(self.service, self.name): # Create it - pgs = self.get_pgs(self.replicas) - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)] + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num)] try: check_call(cmd) except CalledProcessError: @@ -241,7 +248,7 @@ class ErasurePool(Pool): pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m'])) # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), str(pgs), 'erasure', self.erasure_code_profile] try: check_call(cmd) @@ -322,7 +329,8 @@ def set_pool_quota(service, pool_name, max_bytes): :return: None. Can raise CalledProcessError """ # Set a byte quota on a RADOS pool in ceph. - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes] + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, + 'max_bytes', str(max_bytes)] try: check_call(cmd) except CalledProcessError: @@ -343,7 +351,25 @@ def remove_pool_quota(service, pool_name): raise -def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host', +def remove_erasure_profile(service, profile_name): + """ + Create a new erasure code profile if one does not already exist for it. Updates + the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + for more details + :param service: six.string_types. The Ceph user name to run the command under + :param profile_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', + profile_name] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', + failure_domain='host', data_chunks=2, coding_chunks=1, locality=None, durability_estimator=None): """ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2cc5f64 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +PyYAML>=3.1.0 +simplejson>=2.2.0 +netifaces>=0.10.4 +netaddr>=0.7.12,!=0.7.16 +Jinja2>=2.6 # BSD License (3 clause) +six>=1.9.0 +dnspython>=1.12.0 +psutil>=1.1.1,<2.0.0 +python-neutronclient>=2.6.0 diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..3af44d7 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,8 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +coverage>=3.6 +mock>=1.2 +flake8>=2.2.4,<=2.4.1 +os-testr>=0.4.1 +charm-tools