diff --git a/README.md b/README.md index 16f542f..c3f14ba 100644 --- a/README.md +++ b/README.md @@ -76,3 +76,9 @@ Simplestreams. It defaults to settings for downloading images from cloud-images.ubuntu.com, and is not yet tested with other mirror locations. If you have set up your own Simplestreams mirror, you should be able to set the necessary configuration values. + +## `ssl_ca` + +This is used, optionally, to verify the certificates when in ssl mode for +keystone and glance. This should be provided as a base64 encoded PEM +certificate. diff --git a/config.yaml b/config.yaml index f84e678..e7428f7 100644 --- a/config.yaml +++ b/config.yaml @@ -68,6 +68,12 @@ options: default: openstack type: string description: RabbitMQ virtual host to request access on rabbitmq-server. + ssl_ca: + type: string + default: + description: | + base64-encoded SSL CA to use to verify certificates from keystone and + glance if using SSL on the services. nagios_context: default: "juju" type: string diff --git a/hooks/hooks.py b/hooks/hooks.py index 1d542d3..13efe21 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -68,6 +68,20 @@ class UnitNameContext(OSContextGenerator): return {'unit_name': hookenv.local_unit()} +class SSLIdentityServiceContext(IdentityServiceContext): + """Modify the IdentityServiceContext to includea an SSL option. + + This is just a simple way of getting the CA to the + glance-simplestreams-sync.py script. + """ + def __call__(self): + ctxt = super(SSLIdentityServiceContext, self).__call__() + ssl_ca = hookenv.config('ssl_ca') + if ctxt and ssl_ca: + ctxt['ssl_ca'] = ssl_ca + return ctxt + + class MirrorsConfigServiceContext(OSContextGenerator): """Context for mirrors.yaml template. @@ -130,7 +144,7 @@ def get_configs(): openstack_release=get_release()) configs.register(MIRRORS_CONF_FILE_NAME, [MirrorsConfigServiceContext()]) - configs.register(ID_CONF_FILE_NAME, [IdentityServiceContext(), + configs.register(ID_CONF_FILE_NAME, [SSLIdentityServiceContext(), AMQPContext(), UnitNameContext()]) return configs diff --git a/scripts/glance-simplestreams-sync.py b/scripts/glance-simplestreams-sync.py index b7b1713..5655c3b 100755 --- a/scripts/glance-simplestreams-sync.py +++ b/scripts/glance-simplestreams-sync.py @@ -23,6 +23,7 @@ # juju relation to keystone. However, it does not execute in a # juju hook context itself. +import base64 import copy import logging import os @@ -87,6 +88,8 @@ PRODUCT_STREAMS_SERVICE_DESC = 'Ubuntu Product Streams' CRON_POLL_FILENAME = '/etc/cron.d/glance_simplestreams_sync_fastpoll' +CACERT_FILE = os.path.join(CONF_FILE_DIR, 'cacert.pem') + # TODOs: # - allow people to specify their own policy, since they can specify # their own mirrors. @@ -178,7 +181,7 @@ def get_conf(): def get_keystone_client(api_version): if api_version == 3: - ksc = keystone_v3_client.Client( + ksc_vars = dict( auth_url=os.environ['OS_AUTH_URL'], username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'], @@ -186,13 +189,20 @@ def get_keystone_client(api_version): project_domain_name=os.environ['OS_PROJECT_DOMAIN_NAME'], project_name=os.environ['OS_PROJECT_NAME'], project_id=os.environ['OS_PROJECT_ID']) + ksc_class = keystone_v3_client.Client else: - ksc = keystone_client.Client(username=os.environ['OS_USERNAME'], - password=os.environ['OS_PASSWORD'], - tenant_id=os.environ['OS_TENANT_ID'], - tenant_name=os.environ['OS_TENANT_NAME'], - auth_url=os.environ['OS_AUTH_URL']) - return ksc + ksc_vars = dict( + username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + tenant_id=os.environ['OS_TENANT_ID'], + tenant_name=os.environ['OS_TENANT_NAME'], + auth_url=os.environ['OS_AUTH_URL']) + ksc_class = keystone_client.Client + os_cacert = os.environ.get('OS_CACERT', None) + if (os.environ['OS_AUTH_URL'].startswith('https') and + os_cacert is not None): + ksc_vars['cacert'] = os_cacert + return ksc_class(**ksc_vars) def set_openstack_env(id_conf, charm_conf): @@ -206,6 +216,11 @@ def set_openstack_env(id_conf, charm_conf): os.environ['OS_USERNAME'] = id_conf['admin_user'] os.environ['OS_PASSWORD'] = id_conf['admin_password'] os.environ['OS_REGION_NAME'] = charm_conf['region'] + ssl_ca = id_conf.get('ssl_ca', None) + if id_conf['service_protocol'] == 'https' and ssl_ca is not None: + os.environ['OS_CACERT'] = CACERT_FILE + with open(CACERT_FILE, "w") as f: + f.write(base64.b64decode(ssl_ca)) if version == 'v3': # Keystone charm puts all service users in the default domain. # Even so, it would be better if keystone passed this information diff --git a/templates/identity.yaml b/templates/identity.yaml index 2912ee4..94aaee8 100644 --- a/templates/identity.yaml +++ b/templates/identity.yaml @@ -10,6 +10,9 @@ admin_tenant_id: {{ admin_tenant_id }} admin_tenant_name: {{ admin_tenant_name }} admin_user: {{ admin_user }} admin_password: {{ admin_password }} +{% if ssl_ca -%} +ssl_ca: {{ ssl_ca }} +{% endif -%} {% if api_version == '3' -%} admin_domain_name: {{ admin_domain_name }} diff --git a/tests/basic_deployment_ssl.py b/tests/basic_deployment_ssl.py new file mode 100644 index 0000000..c658c58 --- /dev/null +++ b/tests/basic_deployment_ssl.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Basic glance-simplestreams-sync functional tests. +""" + +import base64 +import os +import re +import tempfile + +from charmhelpers.contrib.openstack.amulet.deployment import ( + OpenStackAmuletDeployment +) + +from charmhelpers.contrib.openstack.amulet.utils import ( + OpenStackAmuletUtils, + DEBUG, + # ERROR +) + +import generate_certs + +# Use DEBUG to turn on debug logging +u = OpenStackAmuletUtils(DEBUG) + + +class GlanceBasicDeployment(OpenStackAmuletDeployment): + """Amulet tests on a basic file-backed glance deployment. Verify + relations, service status, endpoint service catalog, create and + delete new image.""" + + SERVICES = ('apache2', 'haproxy', 'glance-api', 'glance-registry') + + def __init__(self, series=None, openstack=None, source=None, + stable=False): + """Deploy the entire test environment.""" + super(GlanceBasicDeployment, self).__init__(series, openstack, + source, stable) + self._add_services() + self._add_relations() + self._configure_services() + self._deploy() + + u.log.info('Waiting on extended status checks...') + + # NOTE(thedac): This charm has a non-standard workload status. + # The default match for ready will fail. Check the other charms + # for standard workload status and check this charm for Sync + # completed. + + # Check for ready + exclude_services = ['glance-simplestreams-sync'] + self._auto_wait_for_status(exclude_services=exclude_services) + + # Check for Sync completed; if SSL is okay, this should work + self._auto_wait_for_status(re.compile('Sync completed.*', + re.IGNORECASE), + include_only=exclude_services) + + self.d.sentry.wait() + + def _assert_services(self, should_run): + u.get_unit_process_ids( + {self.glance_sentry: self.SERVICES}, + expect_success=should_run) + + def _add_services(self): + """Add services + + Add the services that we're testing, where glance is local, + and the rest of the service are from lp branches that are + compatible with the local charm (e.g. stable or next). + """ + this_service = {'name': 'glance-simplestreams-sync'} + other_services = [ + {'name': 'percona-cluster', 'constraints': {'mem': '3072M'}}, + {'name': 'glance'}, + {'name': 'rabbitmq-server'}, + {'name': 'keystone'}, + ] + super(GlanceBasicDeployment, self)._add_services( + this_service, + other_services, + use_source=['glance-simplestreams-sync'], + ) + + def _add_relations(self): + """Add relations for the services.""" + relations = { + 'glance:identity-service': 'keystone:identity-service', + 'glance:shared-db': 'percona-cluster:shared-db', + 'keystone:shared-db': 'percona-cluster:shared-db', + 'glance:amqp': 'rabbitmq-server:amqp', + 'glance-simplestreams-sync:identity-service': + 'keystone:identity-service', + 'glance-simplestreams-sync:amqp': + 'rabbitmq-server:amqp', + } + + super(GlanceBasicDeployment, self)._add_relations(relations) + + def _configure_services(self): + """Configure all of the services.""" + _path = tempfile.gettempdir() + generate_certs.generate_certs(_path) + + _cacert = self.load_base64(_path, 'cacert.pem') + _cert = self.load_base64(_path, 'cert.pem') + _key = self.load_base64(_path, 'cert.key') + + gss_config = { + # https://bugs.launchpad.net/bugs/1686437 + 'source': 'ppa:simplestreams-dev/trunk', + 'use_swift': 'False', + 'ssl_ca': _cacert, + } + glance_config = { + 'ssl_ca': _cacert, + 'ssl_cert': _cert, + 'ssl_key': _key, + } + keystone_config = { + 'admin-password': 'openstack', + 'admin-token': 'ubuntutesting', + 'ssl_ca': _cacert, + 'ssl_cert': _cert, + 'ssl_key': _key, + } + pxc_config = { + 'dataset-size': '25%', + 'max-connections': 1000, + 'root-password': 'ChangeMe123', + 'sst-password': 'ChangeMe123', + } + rabbitmq_server_config = { + 'ssl': 'on', + } + configs = { + 'glance-simplestreams-sync': gss_config, + 'glance': glance_config, + 'keystone': keystone_config, + 'percona-cluster': pxc_config, + 'rabbitmq-server': rabbitmq_server_config, + } + super(GlanceBasicDeployment, self)._configure_services(configs) + + @staticmethod + def load_base64(*path): + with open(os.path.join(*path)) as f: + return base64.b64encode(f.read()) diff --git a/tests/cert.py b/tests/cert.py new file mode 100644 index 0000000..3edf689 --- /dev/null +++ b/tests/cert.py @@ -0,0 +1,243 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for working with x.509 certificates.""" + +import cryptography +from cryptography.hazmat.primitives.asymmetric import padding, rsa +import cryptography.hazmat.primitives.hashes as hashes +import cryptography.hazmat.primitives.serialization as serialization +import datetime +import ipaddress + + +def generate_cert(common_name, + alternative_names=None, + password=None, + issuer_name=None, + signing_key=None, + signing_key_password=None, + generate_ca=False): + """Generate x.509 certificate. + + Example of how to create a certificate chain:: + + (cakey, cacert) = generate_cert( + 'DivineAuthority', + generate_ca=True) + (crkey, crcert) = generate_cert( + 'test.com', + issuer_name='DivineAuthority', + signing_key=cakey) + + :param common_name: Common Name to use in generated certificate + :type common_name: str + :param alternative_names: List of names to add as SubjectAlternativeName + :type alternative_names: Optional[list(str)] + :param password: Password to protect encrypted private key with + :type password: Optional[str] + :param issuer_name: Issuer name, must match provided_private_key issuer + :type issuer_name: Optional[str] + :param signing_key: PEM encoded PKCS8 formatted private key + :type signing_key: Optional[str] + :param signing_key_password: Password to decrypt private key + :type signing_key_password: Optional[str] + :param generate_ca: Generate a certificate usable as a CA certificate + :type generate_ca: bool + :returns: x.509 certificate + :rtype: cryptography.x509.Certificate + """ + if password is not None: + encryption_algorithm = serialization.BestAvailableEncryption(password) + else: + encryption_algorithm = serialization.NoEncryption() + + if signing_key: + _signing_key = serialization.load_pem_private_key( + signing_key, + password=signing_key_password, + backend=cryptography.hazmat.backends.default_backend(), + ) + + private_key = rsa.generate_private_key( + public_exponent=65537, # per RFC 5280 Appendix C + key_size=2048, + backend=cryptography.hazmat.backends.default_backend() + ) + + public_key = private_key.public_key() + + builder = cryptography.x509.CertificateBuilder() + builder = builder.subject_name(cryptography.x509.Name([ + cryptography.x509.NameAttribute( + cryptography.x509.oid.NameOID.COMMON_NAME, common_name), + ])) + + if issuer_name is None: + issuer_name = common_name + + builder = builder.issuer_name(cryptography.x509.Name([ + cryptography.x509.NameAttribute( + cryptography.x509.oid.NameOID.COMMON_NAME, issuer_name), + ])) + builder = builder.not_valid_before( + datetime.datetime.today() - datetime.timedelta(1, 0, 0), + ) + builder = builder.not_valid_after( + datetime.datetime.today() + datetime.timedelta(1, 0, 0), + ) + builder = builder.serial_number(cryptography.x509.random_serial_number()) + builder = builder.public_key(public_key) + + san_list = [cryptography.x509.DNSName(common_name)] + if alternative_names is not None: + for name in alternative_names: + try: + addr = ipaddress.ip_address(name) + except ValueError: + san_list.append(cryptography.x509.DNSName(name)) + else: + san_list.append(cryptography.x509.IPAddress(addr)) + + builder = builder.add_extension( + cryptography.x509.SubjectAlternativeName( + san_list, + ), + critical=False, + ) + builder = builder.add_extension( + cryptography.x509.BasicConstraints(ca=generate_ca, path_length=None), + critical=True, + ) + + if signing_key: + sign_key = _signing_key + else: + sign_key = private_key + + certificate = builder.sign( + private_key=sign_key, + algorithm=cryptography.hazmat.primitives.hashes.SHA256(), + backend=cryptography.hazmat.backends.default_backend(), + ) + + return ( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=encryption_algorithm), + certificate.public_bytes( + serialization.Encoding.PEM) + ) + + +def sign_csr(csr, ca_private_key, ca_cert=None, issuer_name=None, + ca_private_key_password=None, generate_ca=False): + """Sign CSR with the given key. + + :param csr: Certificate to sign + :type csr: str + :param ca_private_key: Private key to be used to sign csr + :type ca_private_key: str + :param ca_cert: Cert to base some options from + :type ca_cert: str + :param issuer_name: Issuer name, must match provided_private_key issuer + :type issuer_name: Optional[str] + :param ca_private_key_password: Password to decrypt ca_private_key + :type ca_private_key_password: Optional[str] + :param generate_ca: Allow resulting cert to be used as ca + :type generate_ca: bool + :returns: x.509 certificate + :rtype: cryptography.x509.Certificate + """ + backend = cryptography.hazmat.backends.default_backend() + # Create x509 artifacts + root_ca_pkey = serialization.load_pem_private_key( + ca_private_key.encode(), + password=ca_private_key_password, + backend=backend) + + new_csr = cryptography.x509.load_pem_x509_csr( + csr.encode(), + backend) + + if ca_cert: + root_ca_cert = cryptography.x509.load_pem_x509_certificate( + ca_cert.encode(), + backend) + issuer_name = root_ca_cert.subject + else: + issuer_name = issuer_name + # Create builder + builder = cryptography.x509.CertificateBuilder() + builder = builder.serial_number( + cryptography.x509.random_serial_number()) + builder = builder.issuer_name(issuer_name) + builder = builder.not_valid_before( + datetime.datetime.today() - datetime.timedelta(1, 0, 0), + ) + builder = builder.not_valid_after( + datetime.datetime.today() + datetime.timedelta(80, 0, 0), + ) + builder = builder.subject_name(new_csr.subject) + builder = builder.public_key(new_csr.public_key()) + + builder = builder.add_extension( + cryptography.x509.BasicConstraints(ca=generate_ca, path_length=None), + critical=True + ) + + # Sign the csr + signer_ca_cert = builder.sign( + private_key=root_ca_pkey, + algorithm=hashes.SHA256(), + backend=backend) + + return signer_ca_cert.public_bytes(encoding=serialization.Encoding.PEM) + + +def is_keys_valid(public_key_string, private_key_string): + """Test whether these are a valid public/private key pair. + + :param public_key_string: PEM encoded key data. + :type public_key_string: str + :param private_key_string: OpenSSH encoded key data. + :type private_key_string: str + """ + private_key = serialization.load_pem_private_key( + private_key_string.encode(), + password=None, + backend=cryptography.hazmat.backends.default_backend() + ) + public_key = serialization.load_ssh_public_key( + public_key_string.encode(), + backend=cryptography.hazmat.backends.default_backend() + ) + message = b"encrypted data" + ciphertext = public_key.encrypt( + message, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None)) + + try: + plaintext = private_key.decrypt( + ciphertext, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None)) + except ValueError: + plaintext = '' + return plaintext == message diff --git a/tests/dev-basic-xenial-pike-ssl b/tests/dev-basic-xenial-pike-ssl new file mode 100755 index 0000000..e4410dd --- /dev/null +++ b/tests/dev-basic-xenial-pike-ssl @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Amulet tests on a basic glance deployment on xenial-pike.""" + +from basic_deployment_ssl import GlanceBasicDeployment + +if __name__ == '__main__': + deployment = GlanceBasicDeployment(series='xenial', + openstack='cloud:xenial-pike', + source='cloud:xenial-updates/pike') + deployment.run_tests() diff --git a/tests/gate-basic-xenial-pike b/tests/gate-basic-xenial-pike index 2f91304..188259e 100755 --- a/tests/gate-basic-xenial-pike +++ b/tests/gate-basic-xenial-pike @@ -20,6 +20,6 @@ from basic_deployment import GlanceBasicDeployment if __name__ == '__main__': deployment = GlanceBasicDeployment(series='xenial', - openstack='cloud:xenial-pike', - source='cloud:xenial-updates/pike') + openstack='cloud:xenial-pike', + source='cloud:xenial-updates/pike') deployment.run_tests() diff --git a/tests/generate_certs.py b/tests/generate_certs.py new file mode 100755 index 0000000..f8099d3 --- /dev/null +++ b/tests/generate_certs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import ipaddress +import itertools +import os +import socket +import tempfile + +import six + +import cert as _cert + +ISSUER_NAME = u'OSCI' + +CERT_DIR = tempfile.gettempdir() + + +def determine_CIDR_EXT(): + ip = socket.gethostbyname(socket.getfqdn()) + if ip.startswith('10.5'): + # running in a bastion + return u"10.5.0.0/24" + else: + # running on UOSCI + return u"172.17.107.0/24" + + +def write_cert(path, filename, data, mode=0o600): + """ + Helper function for writing certificate data to disk. + + :param path: Directory file should be put in + :type path: str + :param filename: Name of file + :type filename: str + :param data: Data to write + :type data: any + :param mode: Create mode (permissions) of file + :type mode: Octal(int) + """ + with os.fdopen(os.open(os.path.join(path, filename), + os.O_WRONLY | os.O_CREAT, mode), 'wb') as f: + f.write(data) + + +# We need to restrain the number of SubjectAlternativeNames we attempt to put +# in the certificate. There is a hard limit for what length the sum of all +# extensions in the certificate can have. +# +# - 2^11 ought to be enough for anybody +def generate_certs(cert_dir=CERT_DIR): + alt_names = [] + for addr in itertools.islice( + ipaddress.IPv4Network(determine_CIDR_EXT()), 2**11): + + if six.PY2: + alt_names.append(unicode(addr)) # NOQA -- py3 doesn't have unicode + else: + alt_names.append(str(addr)) + + (cakey, cacert) = _cert.generate_cert(ISSUER_NAME, + generate_ca=True) + (key, cert) = _cert.generate_cert(u'*.serverstack', + alternative_names=alt_names, + issuer_name=ISSUER_NAME, + signing_key=cakey) + + write_cert(cert_dir, 'cacert.pem', cacert) + write_cert(cert_dir, 'ca.key', cakey) + write_cert(cert_dir, 'cert.pem', cert) + write_cert(cert_dir, 'cert.key', key) + + +if __name__ == '__main__': + generate_certs() diff --git a/tox.ini b/tox.ini index 4a1ca21..3ba8b17 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,8 @@ deps = -r{toxinidir}/requirements.txt basepython = python3.5 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +# charm is NOT Py3 compatible +commands = /bin/true [testenv:py36] basepython = python3.6 @@ -86,6 +88,16 @@ deps = -r{toxinidir}/requirements.txt commands = bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy +[testenv:func27-smoke-ssl] +# Charm functional test, minimal, model setup using SSL - no basic_deployment tests as +# Amulet doesn't do SSL, and basic deployment tests the actual functionality. +# This just tests that the SSL verification bits get to the right places. +basepython = python2.7 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bundletester -vl DEBUG -r json -o func-results.json dev-basic-xenial-pike-ssl --no-destroy + [flake8] ignore = E402,E226 exclude = */charmhelpers