diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index 6a0ba837..2b0a562e 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils): else: msg = 'No message retrieved.' amulet.raise_status(amulet.FAIL, msg) + + def validate_memcache(self, sentry_unit, conf, os_release, + earliest_release=5, section='keystone_authtoken', + check_kvs=None): + """Check Memcache is running and is configured to be used + + Example call from Amulet test: + + def test_110_memcache(self): + u.validate_memcache(self.neutron_api_sentry, + '/etc/neutron/neutron.conf', + self._get_openstack_release()) + + :param sentry_unit: sentry unit + :param conf: OpenStack config file to check memcache settings + :param os_release: Current OpenStack release int code + :param earliest_release: Earliest Openstack release to check int code + :param section: OpenStack config file section to check + :param check_kvs: Dict of settings to check in config file + :returns: None + """ + if os_release < earliest_release: + self.log.debug('Skipping memcache checks for deployment. {} <' + 'mitaka'.format(os_release)) + return + _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'} + self.log.debug('Checking memcached is running') + ret = self.validate_services_by_name({sentry_unit: ['memcached']}) + if ret: + amulet.raise_status(amulet.FAIL, msg='Memcache running check' + 'failed {}'.format(ret)) + else: + self.log.debug('OK') + self.log.debug('Checking memcache url is configured in {}'.format( + conf)) + if self.validate_config_data(sentry_unit, conf, section, _kvs): + message = "Memcache config error in: {}".format(conf) + amulet.raise_status(amulet.FAIL, msg=message) + else: + self.log.debug('OK') + self.log.debug('Checking memcache configuration in ' + '/etc/memcached.conf') + contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf', + fatal=True) + ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs') + if ubuntu_release <= 'trusty': + memcache_listen_addr = 'ip6-localhost' + else: + memcache_listen_addr = '::1' + expected = { + '-p': '11211', + '-l': memcache_listen_addr} + found = [] + for key, value in expected.items(): + for line in contents.split('\n'): + if line.startswith(key): + self.log.debug('Checking {} is set to {}'.format( + key, + value)) + assert value == line.split()[-1] + self.log.debug(line.split()[-1]) + found.append(key) + if sorted(found) == sorted(expected.keys()): + self.log.debug('OK') + else: + message = "Memcache config error in: /etc/memcached.conf" + amulet.raise_status(amulet.FAIL, msg=message) diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index d5b3a33b..91ddcecd 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -90,6 +90,7 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import ( config_flags_parser, get_host_ip, + enable_memcache, ) from charmhelpers.core.unitdata import kv @@ -1512,3 +1513,27 @@ class AppArmorContext(OSContextGenerator): "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode'])) raise e + + +class MemcacheContext(OSContextGenerator): + """Memcache context + + This context provides options for configuring a local memcache client and + server + """ + def __call__(self): + ctxt = {} + ctxt['use_memcache'] = enable_memcache(config('openstack-origin')) + if ctxt['use_memcache']: + # Trusty version of memcached does not support ::1 as a listen + # address so use host file entry instead + if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty': + ctxt['memcache_server'] = '::1' + else: + ctxt['memcache_server'] = 'ip6-localhost' + ctxt['memcache_server_formatted'] = '[::1]' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = 'inet6:{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) + return ctxt diff --git a/charmhelpers/contrib/openstack/templates/memcached.conf b/charmhelpers/contrib/openstack/templates/memcached.conf new file mode 100644 index 00000000..26cb037c --- /dev/null +++ b/charmhelpers/contrib/openstack/templates/memcached.conf @@ -0,0 +1,53 @@ +############################################################################### +# [ WARNING ] +# memcached configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### + +# memcached default config file +# 2003 - Jay Bonci +# This configuration file is read by the start-memcached script provided as +# part of the Debian GNU/Linux distribution. + +# Run memcached as a daemon. This command is implied, and is not needed for the +# daemon to run. See the README.Debian that comes with this package for more +# information. +-d + +# Log memcached's output to /var/log/memcached +logfile /var/log/memcached.log + +# Be verbose +# -v + +# Be even more verbose (print client commands as well) +# -vv + +# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default +# Note that the daemon will grow to this size, but does not start out holding this much +# memory +-m 64 + +# Default connection port is 11211 +-p {{ memcache_port }} + +# Run the daemon as root. The start-memcached will default to running as root if no +# -u command is present in this config file +-u memcache + +# Specify which IP address to listen on. The default is to listen on all IP addresses +# This parameter is one of the only security measures that memcached has, so make sure +# it's listening on a firewalled interface. +-l {{ memcache_server }} + +# Limit the number of simultaneous incoming connections. The daemon default is 1024 +# -c 1024 + +# Lock down all paged memory. Consult with the README and homepage before you do this +# -k + +# Return error when memory is exhausted (rather than removing items) +# -M + +# Maximize core file limit +# -r diff --git a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 7c6f0c35..8e6889e0 100644 --- a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -14,4 +14,7 @@ project_name = {{ admin_tenant_name }} username = {{ admin_user }} password = {{ admin_password }} signing_dir = {{ signing_dir }} +{% if use_memcache == true %} +memcached_servers = {{ memcache_url }} +{% endif -%} {% endif -%} diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 553ff8fe..59f9f512 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -1925,3 +1925,28 @@ def os_application_version_set(package): application_version_set(os_release(package)) else: application_version_set(application_version) + + +def enable_memcache(source=None, release=None): + """Determine if memcache should be enabled on the local unit + + @param source: source string for charm + @param release: release of OpenStack currently deployed + @returns boolean Whether memcache should be enabled + """ + if not release: + release = get_os_codename_install_source(source) + return release >= 'mitaka' + + +def token_cache_pkgs(source=None, release=None): + """Determine additional packages needed for token caching + + @param source: source string for charm + @param release: release of OpenStack currently deployed + @returns List of package to enable token caching + """ + packages = [] + if enable_memcache(source=source, release=release): + packages.extend(['memcached', 'python-memcache']) + return packages diff --git a/hooks/glance_utils.py b/hooks/glance_utils.py index 7bee5399..03ae9220 100644 --- a/hooks/glance_utils.py +++ b/hooks/glance_utils.py @@ -82,6 +82,8 @@ from charmhelpers.contrib.openstack.utils import ( resume_unit, incomplete_relation_data, os_application_version_set, + token_cache_pkgs, + enable_memcache, ) from charmhelpers.core.templating import render @@ -144,6 +146,7 @@ HAPROXY_CONF = "/etc/haproxy/haproxy.cfg" HTTPS_APACHE_CONF = "/etc/apache2/sites-available/openstack_https_frontend" HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \ "openstack_https_frontend.conf" +MEMCACHED_CONF = '/etc/memcached.conf' TEMPLATES = 'templates/' @@ -171,7 +174,8 @@ CONFIG_FILES = OrderedDict([ context.WorkerConfigContext(), context.OSConfigFlagContext( charm_flag='registry-config-flags', - template_flag='registry_config_flags')], + template_flag='registry_config_flags'), + context.MemcacheContext()], 'services': ['glance-registry'] }), (GLANCE_API_CONF, { @@ -197,7 +201,8 @@ CONFIG_FILES = OrderedDict([ context.SubordinateConfigContext( interface=['storage-backend'], service=['glance-api'], - config_file=GLANCE_API_CONF)], + config_file=GLANCE_API_CONF), + context.MemcacheContext()], 'services': ['glance-api'] }), (ceph_config_file(), { @@ -256,6 +261,8 @@ def register_configs(): configs.register(HTTPS_APACHE_CONF, CONFIG_FILES[HTTPS_APACHE_CONF]['hook_contexts']) + if enable_memcache(release=release): + configs.register(MEMCACHED_CONF, [context.MemcacheContext()]) return configs @@ -266,6 +273,7 @@ def determine_packages(): packages |= set(BASE_GIT_PACKAGES) packages -= set(GIT_PACKAGE_BLACKLIST) + packages |= set(token_cache_pkgs(source=config('openstack-origin'))) return sorted(packages) @@ -328,6 +336,10 @@ def restart_map(): svcs.append(svc) if svcs: _map.append((f, svcs)) + + if enable_memcache(source=config('openstack-origin')): + _map.append((MEMCACHED_CONF, ['memcached'])) + return OrderedDict(_map) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index b3196af2..9f1d26a4 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -251,6 +251,16 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment): if ret: amulet.raise_status(amulet.FAIL, msg=ret) + def test_115_memcache(self): + u.validate_memcache(self.glance_sentry, + '/etc/glance/glance-api.conf', + self._get_openstack_release(), + earliest_release=self.trusty_mitaka) + u.validate_memcache(self.glance_sentry, + '/etc/glance/glance-registry.conf', + self._get_openstack_release(), + earliest_release=self.trusty_mitaka) + def test_200_mysql_glance_db_relation(self): """Verify the mysql:glance shared-db relation data""" u.log.debug('Checking mysql to glance shared-db relation data...') diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 8e13ab14..f9e4c3af 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -148,7 +148,8 @@ class AmuletUtils(object): for service_name in services_list: if (self.ubuntu_releases.index(release) >= systemd_switch or - service_name in ['rabbitmq-server', 'apache2']): + service_name in ['rabbitmq-server', 'apache2', + 'memcached']): # init is systemd (or regular sysv) cmd = 'sudo service {} status'.format(service_name) output, code = sentry_unit.run(cmd) diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 6a0ba837..2b0a562e 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -1133,3 +1133,70 @@ class OpenStackAmuletUtils(AmuletUtils): else: msg = 'No message retrieved.' amulet.raise_status(amulet.FAIL, msg) + + def validate_memcache(self, sentry_unit, conf, os_release, + earliest_release=5, section='keystone_authtoken', + check_kvs=None): + """Check Memcache is running and is configured to be used + + Example call from Amulet test: + + def test_110_memcache(self): + u.validate_memcache(self.neutron_api_sentry, + '/etc/neutron/neutron.conf', + self._get_openstack_release()) + + :param sentry_unit: sentry unit + :param conf: OpenStack config file to check memcache settings + :param os_release: Current OpenStack release int code + :param earliest_release: Earliest Openstack release to check int code + :param section: OpenStack config file section to check + :param check_kvs: Dict of settings to check in config file + :returns: None + """ + if os_release < earliest_release: + self.log.debug('Skipping memcache checks for deployment. {} <' + 'mitaka'.format(os_release)) + return + _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'} + self.log.debug('Checking memcached is running') + ret = self.validate_services_by_name({sentry_unit: ['memcached']}) + if ret: + amulet.raise_status(amulet.FAIL, msg='Memcache running check' + 'failed {}'.format(ret)) + else: + self.log.debug('OK') + self.log.debug('Checking memcache url is configured in {}'.format( + conf)) + if self.validate_config_data(sentry_unit, conf, section, _kvs): + message = "Memcache config error in: {}".format(conf) + amulet.raise_status(amulet.FAIL, msg=message) + else: + self.log.debug('OK') + self.log.debug('Checking memcache configuration in ' + '/etc/memcached.conf') + contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf', + fatal=True) + ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs') + if ubuntu_release <= 'trusty': + memcache_listen_addr = 'ip6-localhost' + else: + memcache_listen_addr = '::1' + expected = { + '-p': '11211', + '-l': memcache_listen_addr} + found = [] + for key, value in expected.items(): + for line in contents.split('\n'): + if line.startswith(key): + self.log.debug('Checking {} is set to {}'.format( + key, + value)) + assert value == line.split()[-1] + self.log.debug(line.split()[-1]) + found.append(key) + if sorted(found) == sorted(expected.keys()): + self.log.debug('OK') + else: + message = "Memcache config error in: /etc/memcached.conf" + amulet.raise_status(amulet.FAIL, msg=message) diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py index c3d7b7f2..7aa0a823 100644 --- a/unit_tests/test_actions_git_reinstall.py +++ b/unit_tests/test_actions_git_reinstall.py @@ -31,7 +31,8 @@ with patch('actions.hooks.charmhelpers.contrib.hardening.harden.harden') as \ lambda *args, **kwargs: f(*args, **kwargs)) with patch('actions.hooks.charmhelpers.core.hookenv.config') as config: with patch('actions.hooks.glance_utils.register_configs'): - from actions import git_reinstall + with patch('actions.hooks.glance_utils.restart_map'): + from actions import git_reinstall TO_PATCH = [ 'config', diff --git a/unit_tests/test_glance_relations.py b/unit_tests/test_glance_relations.py index 99a6a228..b6fbce8b 100644 --- a/unit_tests/test_glance_relations.py +++ b/unit_tests/test_glance_relations.py @@ -109,8 +109,11 @@ class GlanceRelationTests(CharmTestCase): self.config.side_effect = self.test_config.get self.network_get_primary_address.side_effect = NotImplementedError + @patch.object(utils, 'config') + @patch.object(utils, 'token_cache_pkgs') @patch.object(utils, 'git_install_requested') - def test_install_hook(self, git_requested): + def test_install_hook(self, git_requested, token_cache_pkgs, util_config): + token_cache_pkgs.return_value = ['memcached'] git_requested.return_value = False repo = 'cloud:precise-grizzly' self.test_config.set('openstack-origin', repo) @@ -119,14 +122,18 @@ class GlanceRelationTests(CharmTestCase): self.configure_installation_source.assert_called_with(repo) self.apt_update.assert_called_with(fatal=True) self.apt_install.assert_called_with( - ['apache2', 'glance', 'haproxy', 'python-keystone', + ['apache2', 'glance', 'haproxy', 'memcached', 'python-keystone', 'python-mysqldb', 'python-psycopg2', 'python-six', 'python-swiftclient', 'uuid'], fatal=True) self.assertTrue(self.execd_preinstall.called) self.git_install.assert_called_with(None) + @patch.object(utils, 'config') + @patch.object(utils, 'token_cache_pkgs') @patch.object(utils, 'git_install_requested') - def test_install_hook_precise_distro(self, git_requested): + def test_install_hook_precise_distro(self, git_requested, token_cache_pkgs, + util_config): + token_cache_pkgs.return_value = [] git_requested.return_value = False self.test_config.set('openstack-origin', 'distro') self.lsb_release.return_value = {'DISTRIB_RELEASE': 12.04, @@ -137,8 +144,11 @@ class GlanceRelationTests(CharmTestCase): "cloud:precise-folsom" ) + @patch.object(utils, 'config') + @patch.object(utils, 'token_cache_pkgs') @patch.object(utils, 'git_install_requested') - def test_install_hook_git(self, git_requested): + def test_install_hook_git(self, git_requested, token_cache_pkgs, + util_config): git_requested.return_value = True repo = 'cloud:trusty-juno' openstack_origin_git = { @@ -702,9 +712,12 @@ class GlanceRelationTests(CharmTestCase): call('/etc/haproxy/haproxy.cfg')], configs.write.call_args_list) + @patch.object(utils, 'config') + @patch.object(utils, 'token_cache_pkgs') @patch.object(relations, 'CONFIGS') @patch.object(utils, 'git_install_requested') - def test_upgrade_charm(self, git_requested, configs): + def test_upgrade_charm(self, git_requested, configs, token_cache_pkgs, + util_config): git_requested.return_value = False self.filter_installed_packages.return_value = ['test'] relations.upgrade_charm() diff --git a/unit_tests/test_glance_utils.py b/unit_tests/test_glance_utils.py index aadf951f..96995704 100644 --- a/unit_tests/test_glance_utils.py +++ b/unit_tests/test_glance_utils.py @@ -49,6 +49,8 @@ TO_PATCH = [ 'install_alternative', 'lsb_release', 'os_application_version_set', + 'enable_memcache', + 'token_cache_pkgs', ] DPKG_OPTS = [ @@ -132,6 +134,8 @@ class TestGlanceUtils(CharmTestCase): self.mkdir.assert_called_with('/etc/ceph') def test_restart_map(self): + self.enable_memcache.return_value = True + self.config.side_effect = None self.service_name.return_value = 'glance' ex_map = OrderedDict([ @@ -140,19 +144,31 @@ class TestGlanceUtils(CharmTestCase): (utils.ceph_config_file(), ['glance-api', 'glance-registry']), (utils.HAPROXY_CONF, ['haproxy']), (utils.HTTPS_APACHE_CONF, ['apache2']), - (utils.HTTPS_APACHE_24_CONF, ['apache2']) + (utils.HTTPS_APACHE_24_CONF, ['apache2']), + (utils.MEMCACHED_CONF, ['memcached']) ]) self.assertEquals(ex_map, utils.restart_map()) + self.enable_memcache.return_value = False + del ex_map[utils.MEMCACHED_CONF] + self.assertEquals(ex_map, utils.restart_map()) + @patch.object(utils, 'token_cache_pkgs') @patch.object(utils, 'git_install_requested') - def test_determine_packages(self, git_install_requested): + def test_determine_packages(self, git_install_requested, token_cache_pkgs): + self.config.side_effect = None + token_cache_pkgs.return_value = [] git_install_requested.return_value = False - result = utils.determine_packages() ex = utils.PACKAGES - self.assertEquals(set(ex), set(result)) + self.assertEquals(set(ex), set(utils.determine_packages())) + token_cache_pkgs.return_value = ['memcached'] + ex.append('memcached') + self.assertEquals(set(ex), set(utils.determine_packages())) + @patch.object(utils, 'token_cache_pkgs') @patch.object(utils, 'git_install_requested') - def test_determine_packages_git(self, git_install_requested): + def test_determine_packages_git(self, git_install_requested, + token_cache_pkgs): + self.config.side_effect = None git_install_requested.return_value = True result = utils.determine_packages() ex = utils.PACKAGES + utils.BASE_GIT_PACKAGES