diff --git a/charms_openstack/adapters.py b/charms_openstack/adapters.py index 88a4509..12e1629 100644 --- a/charms_openstack/adapters.py +++ b/charms_openstack/adapters.py @@ -27,7 +27,9 @@ import charmhelpers.contrib.hahelpers.cluster as ch_cluster import charmhelpers.contrib.network.ip as ch_ip import charmhelpers.contrib.openstack.utils as ch_utils import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as ch_host import charms_openstack.ip as os_ip +import charms_openstack.os_release_data as os_release_data ADDRESS_TYPES = os_ip.ADDRESS_MAP.keys() @@ -801,6 +803,35 @@ class APIConfigurationAdapter(ConfigurationAdapter): eps = [ep[2] for ep in self.endpoints] return sorted(list(set(eps))) + @property + def use_memcache(self): + release = ch_utils.get_os_codename_install_source( + self.openstack_origin) + if release not in os_release_data.KNOWN_RELEASES: + return ValueError("Unkown release {}".format(release)) + return (os_release_data.KNOWN_RELEASES.index(release) >= + os_release_data.KNOWN_RELEASES.index('mitaka')) + + @property + def memcache_server(self): + if ch_host.lsb_release()['DISTRIB_RELEASE'] > '14.04': + memcache_server = '::1' + else: + memcache_server = 'ip6-localhost' + return memcache_server + + @property + def memcache_host(self): + return '[::1]' + + @property + def memcache_port(self): + return '11211' + + @property + def memcache_url(self): + return 'inet6:{}:{}'.format(self.memcache_host, self.memcache_port) + def make_default_relation_adapter(base_cls, relation, properties): """Create a default relation adapter using a base class, and custom diff --git a/charms_openstack/charm.py b/charms_openstack/charm.py index 532a21b..2a0998f 100644 --- a/charms_openstack/charm.py +++ b/charms_openstack/charm.py @@ -42,8 +42,9 @@ import charmhelpers.core.unitdata as unitdata import charmhelpers.fetch as fetch import charms.reactive as reactive -import charms_openstack.ip as os_ip import charms_openstack.adapters as os_adapters +import charms_openstack.ip as os_ip +import charms_openstack.os_release_data as os_release_data # _releases{} is a dictionary of release -> class that is instantiated @@ -63,22 +64,6 @@ _singleton = None # This is to enable the defining code to define which release is used. _release_selector_function = None -# List of releases that OpenStackCharm based charms know about -KNOWN_RELEASES = [ - 'diablo', - 'essex', - 'folsom', - 'grizzly', - 'havana', - 'icehouse', - 'juno', - 'kilo', - 'liberty', - 'mitaka', - 'newton', - 'ocata', -] - VIP_KEY = "vip" CIDR_KEY = "vip_cidr" IFACE_KEY = "vip_iface" @@ -377,18 +362,20 @@ def get_charm_instance(release=None, *args, **kwargs): cls = _releases[known_releases[-1]] else: # check that the release is a valid release - if release not in KNOWN_RELEASES: + if release not in os_release_data.KNOWN_RELEASES: raise RuntimeError( "Release {} is not a known OpenStack release?".format(release)) - release_index = KNOWN_RELEASES.index(release) - if release_index < KNOWN_RELEASES.index(known_releases[0]): + release_index = os_release_data.KNOWN_RELEASES.index(release) + if (release_index < + os_release_data.KNOWN_RELEASES.index(known_releases[0])): raise RuntimeError( "Release {} is not supported by this charm. Earliest support " "is {} release".format(release, known_releases[0])) else: # try to find the release that is supported. for known_release in reversed(known_releases): - if release_index >= KNOWN_RELEASES.index(known_release): + if (release_index >= + os_release_data.KNOWN_RELEASES.index(known_release)): cls = _releases[known_release] break if cls is None: @@ -458,7 +445,7 @@ class OpenStackCharmMeta(type): return if 'release' in members.keys(): release = members['release'] - if release not in KNOWN_RELEASES: + if release not in os_release_data.KNOWN_RELEASES: raise RuntimeError( "Release {} is not a known OpenStack release" .format(release)) @@ -559,6 +546,7 @@ class OpenStackCharm(object): ha_resources = [] adapters_class = None HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' + MEMCACHE_CONF = '/etc/memcached.conf' package_codenames = {} @property @@ -728,7 +716,9 @@ class OpenStackCharm(object): protocol = protocol.lower() else: protocol = '' - lines = [l for l in subprocess.check_output(_args).split() if l] + lines = [l for l in + subprocess.check_output(_args).decode('UTF-8').split() + if l] ports = [] for line in lines: p, p_type = line.split('/') @@ -1245,6 +1235,11 @@ class OpenStackAPICharm(OpenStackCharm): # If None, then the default ConfigurationAdapter is used. configuration_class = os_adapters.APIConfigurationAdapter + def upgrade_charm(self): + """Setup token cache in case previous charm version did not.""" + self.setup_token_cache() + super(OpenStackAPICharm, self).upgrade_charm() + def install(self): """Install packages related to this charm based on contents of self.packages attribute. @@ -1252,6 +1247,36 @@ class OpenStackAPICharm(OpenStackCharm): self.configure_source() super(OpenStackAPICharm, self).install() + def setup_token_cache(self): + """Check if a token cache package is needed and install it if it is""" + if fetch.filter_installed_packages(self.token_cache_pkgs()): + self.install() + + def enable_memcache(self, release=None): + """Determine if memcache should be enabled on the local unit + + @param release: release of OpenStack currently deployed + @returns boolean Whether memcache should be enabled + """ + if not release: + release = os_utils.get_os_codename_install_source( + self.config['openstack-origin']) + if release not in os_release_data.KNOWN_RELEASES: + return ValueError("Unkown release {}".format(release)) + return (os_release_data.KNOWN_RELEASES.index(release) >= + os_release_data.KNOWN_RELEASES.index('mitaka')) + + def token_cache_pkgs(self, release=None): + """Determine additional packages needed for token caching + + @param release: release of OpenStack currently deployed + @returns List of package to enable token caching + """ + packages = [] + if self.enable_memcache(release=release): + packages.extend(['memcached', 'python-memcache']) + return packages + def get_amqp_credentials(self): """Provide the default amqp username and vhost as a tuple. @@ -1290,6 +1315,30 @@ class OpenStackAPICharm(OpenStackCharm): "get_database_setup() needs to be overriden in the derived " "class") + @property + def all_packages(self): + """List of packages to be installed + + @return ['pkg1', 'pkg2', ...] + """ + return (super(OpenStackAPICharm, self).all_packages + + self.token_cache_pkgs()) + + @property + def full_restart_map(self): + """Map of services to be restarted if a file changes + + @return { + 'file1': ['svc1', 'svc3'], + 'file2': ['svc2', 'svc3'], + ... + } + """ + _restart_map = super(OpenStackAPICharm, self).full_restart_map.copy() + if self.enable_memcache(): + _restart_map[self.MEMCACHE_CONF] = ['memcached'] + return _restart_map + class HAOpenStackCharm(OpenStackAPICharm): @@ -1332,7 +1381,7 @@ class HAOpenStackCharm(OpenStackAPICharm): @return ['pkg1', 'pkg2', ...] """ - _packages = self.packages[:] + _packages = super(HAOpenStackCharm, self).all_packages if self.haproxy_enabled(): _packages.append('haproxy') if self.apache_enabled(): @@ -1349,7 +1398,7 @@ class HAOpenStackCharm(OpenStackAPICharm): ... } """ - _restart_map = self.restart_map.copy() + _restart_map = super(HAOpenStackCharm, self).full_restart_map if self.haproxy_enabled(): _restart_map[self.HAPROXY_CONF] = ['haproxy'] if self.apache_enabled(): diff --git a/charms_openstack/os_release_data.py b/charms_openstack/os_release_data.py new file mode 100644 index 0000000..09c27f4 --- /dev/null +++ b/charms_openstack/os_release_data.py @@ -0,0 +1,33 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OpenStackCharm() - base class for build OpenStack charms from for the +# reactive framework. + +# need/want absolute imports for the package imports to work properly + +KNOWN_RELEASES = [ + 'diablo', + 'essex', + 'folsom', + 'grizzly', + 'havana', + 'icehouse', + 'juno', + 'kilo', + 'liberty', + 'mitaka', + 'newton', + 'ocata', +] diff --git a/unit_tests/test_charms_openstack_adapters.py b/unit_tests/test_charms_openstack_adapters.py index efbf9d3..0968653 100644 --- a/unit_tests/test_charms_openstack_adapters.py +++ b/unit_tests/test_charms_openstack_adapters.py @@ -698,6 +698,43 @@ class TestAPIConfigurationAdapter(unittest.TestCase): c = adapters.APIConfigurationAdapter() self.assertEqual(c.determine_service_port(80), 70) + def test_use_memcache(self): + test_config = {'openstack-origin': 'distro'} + with mock.patch.object(adapters.hookenv, 'config', + new=lambda: test_config): + with mock.patch.object(adapters.ch_utils, + 'get_os_codename_install_source', + return_value='liberty'): + c = adapters.APIConfigurationAdapter() + self.assertFalse(c.use_memcache) + with mock.patch.object(adapters.ch_utils, + 'get_os_codename_install_source', + return_value='newton'): + c = adapters.APIConfigurationAdapter() + self.assertTrue(c.use_memcache) + + def test_memcache_server(self): + with mock.patch.object(adapters.ch_host, 'lsb_release', + return_value={'DISTRIB_RELEASE': '14.04'}): + c = adapters.APIConfigurationAdapter() + self.assertEqual(c.memcache_server, 'ip6-localhost') + with mock.patch.object(adapters.ch_host, 'lsb_release', + return_value={'DISTRIB_RELEASE': '16.04'}): + c = adapters.APIConfigurationAdapter() + self.assertEqual(c.memcache_server, '::1') + + def test_memcache_host(self): + self.assertEqual(adapters.APIConfigurationAdapter().memcache_host, + '[::1]') + + def test_memcache_port(self): + self.assertEqual(adapters.APIConfigurationAdapter().memcache_port, + '11211') + + def test_memcache_url(self): + self.assertEqual(adapters.APIConfigurationAdapter().memcache_url, + 'inet6:[::1]:11211') + class FakePeerHARelationAdapter(object): diff --git a/unit_tests/test_charms_openstack_charm.py b/unit_tests/test_charms_openstack_charm.py index 158196c..0fc380c 100644 --- a/unit_tests/test_charms_openstack_charm.py +++ b/unit_tests/test_charms_openstack_charm.py @@ -495,7 +495,7 @@ class TestOpenStackCharm(BaseOpenStackCharmTest): 'filter_installed_packages', name='fip', return_value=None) - self.patch_object(chm.subprocess, 'check_output', return_value='\n') + self.patch_object(chm.subprocess, 'check_output', return_value=b'\n') self.target.install() self.target.set_state.assert_called_once_with('charmname-installed') self.fip.assert_called_once_with([]) @@ -634,20 +634,61 @@ class TestOpenStackAPICharm(BaseOpenStackCharmTest): super(TestOpenStackAPICharm, self).setUp(chm.OpenStackAPICharm, TEST_CONFIG) + def test_upgrade_charm(self): + self.patch_target('setup_token_cache') + self.patch_target('update_api_ports') + self.target.upgrade_charm() + self.target.setup_token_cache.assert_called_once_with() + def test_install(self): # Test set_state and configure_source are called self.patch_target('set_state') self.patch_target('configure_source') + self.patch_target('enable_memcache', return_value=False) self.patch_object(chm.charmhelpers.fetch, 'filter_installed_packages', name='fip', return_value=None) - self.patch_object(chm.subprocess, 'check_output', return_value='\n') + self.patch_object(chm.subprocess, 'check_output', return_value=b'\n') self.target.install() # self.target.set_state.assert_called_once_with('charmname-installed') self.target.configure_source.assert_called_once_with() self.fip.assert_called_once_with([]) + def test_setup_token_cache(self): + self.patch_target('token_cache_pkgs') + self.patch_target('install') + self.patch_object(chm.charmhelpers.fetch, + 'filter_installed_packages', + name='fip', + return_value=['memcached']) + self.target.setup_token_cache() + self.install.assert_called_once_with() + self.fip.return_value = [] + self.install.reset_mock() + self.target.setup_token_cache() + self.assertFalse(self.install.called) + + def test_enable_memcache(self): + self.assertFalse(self.target.enable_memcache(release='liberty')) + self.assertTrue(self.target.enable_memcache(release='newton')) + self.patch_target('config', new={'openstack-origin': 'distro'}) + self.patch_object(chm.os_utils, + 'get_os_codename_install_source', + name='gocis') + self.gocis.return_value = 'liberty' + self.assertFalse(self.target.enable_memcache()) + self.gocis.return_value = 'newton' + self.assertTrue(self.target.enable_memcache()) + + def test_token_cache_pkgs(self): + self.patch_target('enable_memcache') + self.enable_memcache.return_value = True + self.assertEqual(self.target.token_cache_pkgs(), ['memcached', + 'python-memcache']) + self.enable_memcache.return_value = False + self.assertEqual(self.target.token_cache_pkgs(), []) + def test_get_amqp_credentials(self): # verify that the instance throws an error if not overriden with self.assertRaises(RuntimeError): @@ -658,6 +699,29 @@ class TestOpenStackAPICharm(BaseOpenStackCharmTest): with self.assertRaises(RuntimeError): self.target.get_database_setup() + def test_all_packages(self): + self.patch_target('enable_memcache') + self.patch_target('packages', new=['pkg1', 'pkg2']) + self.enable_memcache.return_value = True + self.assertEqual(self.target.all_packages, + ['pkg1', 'pkg2', 'memcached', 'python-memcache']) + self.enable_memcache.return_value = False + self.assertEqual(self.target.all_packages, ['pkg1', 'pkg2']) + + def test_full_restart_map(self): + self.patch_target('enable_memcache') + base_restart_map = { + 'conf1': ['svc1'], + 'conf2': ['svc1']} + self.patch_target('restart_map', new=base_restart_map) + self.enable_memcache.return_value = True + self.assertEqual(self.target.full_restart_map, + {'conf1': ['svc1'], + 'conf2': ['svc1'], + '/etc/memcached.conf': ['memcached']}) + self.enable_memcache.return_value = False + self.assertEqual(self.target.full_restart_map, base_restart_map) + class TestHAOpenStackCharm(BaseOpenStackCharmTest): # Note that this only tests the OpenStackCharm() class, which has not very @@ -668,6 +732,45 @@ class TestHAOpenStackCharm(BaseOpenStackCharmTest): super(TestHAOpenStackCharm, self).setUp(chm.HAOpenStackCharm, TEST_CONFIG) + def test_all_packages(self): + self.patch_target('packages', new=['pkg1']) + self.patch_target('token_cache_pkgs', return_value=[]) + self.patch_target('haproxy_enabled', return_value=False) + self.patch_target('apache_enabled', return_value=False) + self.assertEqual(['pkg1'], self.target.all_packages) + self.token_cache_pkgs.return_value = ['memcache'] + self.haproxy_enabled.return_value = True + self.apache_enabled.return_value = True + self.assertEqual(['pkg1', 'memcache', 'haproxy', 'apache2'], + self.target.all_packages) + + def test_full_restart_map_disabled(self): + base_restart_map = { + 'conf1': ['svc1'], + 'conf2': ['svc1']} + self.patch_target('restart_map', new=base_restart_map) + self.patch_target('enable_memcache', return_value=False) + self.patch_target('haproxy_enabled', return_value=False) + self.patch_target('apache_enabled', return_value=False) + self.assertEqual(base_restart_map, self.target.full_restart_map) + + def test_full_restart_map_enabled(self): + base_restart_map = { + 'conf1': ['svc1'], + 'conf2': ['svc1']} + self.patch_target('restart_map', new=base_restart_map) + self.patch_target('enable_memcache', return_value=True) + self.patch_target('haproxy_enabled', return_value=True) + self.patch_target('apache_enabled', return_value=True) + self.assertEqual( + self.target.full_restart_map, + {'/etc/apache2/sites-available/openstack_https_frontend.conf': + ['apache2'], + '/etc/haproxy/haproxy.cfg': ['haproxy'], + '/etc/memcached.conf': ['memcached'], + 'conf1': ['svc1'], + 'conf2': ['svc1']}) + def test_haproxy_enabled(self): self.patch_target('ha_resources', new=['haproxy']) self.assertTrue(self.target.haproxy_enabled()) @@ -744,13 +847,18 @@ class TestHAOpenStackCharm(BaseOpenStackCharmTest): self.set_state.assert_called_once_with('haproxy.stat.password', mock.ANY) - def test_hacharm_all_packages(self): + def test_hacharm_all_packages_enabled(self): + self.patch_target('enable_memcache', return_value=False) self.patch_target('haproxy_enabled', return_value=True) self.assertTrue('haproxy' in self.target.all_packages) + + def test_hacharm_all_packages_disabled(self): + self.patch_target('enable_memcache', return_value=False) self.patch_target('haproxy_enabled', return_value=False) self.assertFalse('haproxy' in self.target.all_packages) def test_hacharm_full_restart_map(self): + self.patch_target('enable_memcache', return_value=False) self.patch_target('haproxy_enabled', return_value=True) self.assertTrue( self.target.full_restart_map.get( @@ -1093,7 +1201,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest): self.fip.side_effect = lambda x: ['p1', 'p2'] self.patch_object(chm.hookenv, 'status_set') self.patch_object(chm.hookenv, 'apt_install') - self.patch_object(chm.subprocess, 'check_output', return_value='\n') + self.patch_object(chm.subprocess, 'check_output', return_value=b'\n') self.target.install() # TODO: remove next commented line as we don't set this state anymore # self.target.set_state.assert_called_once_with('my-charm-installed') @@ -1115,7 +1223,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest): def test_update_api_ports(self): self.patch_object(chm.hookenv, 'open_port') self.patch_object(chm.hookenv, 'close_port') - self.patch_object(chm.subprocess, 'check_output', return_value='\n') + self.patch_object(chm.subprocess, 'check_output', return_value=b'\n') self.target.api_ports = { 'api': { 'public': 1, @@ -1136,7 +1244,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest): # that should be closed self.open_port.reset_mock() self.close_port.reset_mock() - self.check_output.return_value = "1/tcp\n2/tcp\n3/udp\n4/tcp\n" + self.check_output.return_value = b"1/tcp\n2/tcp\n3/udp\n4/tcp\n" # port 3 should be opened, port 4 should be closed. open_calls = [mock.call(3)] close_calls = [mock.call(4)] @@ -1146,9 +1254,9 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest): def test_opened_ports(self): self.patch_object(chm.subprocess, 'check_output') - self.check_output.return_value = '\n' + self.check_output.return_value = b'\n' self.assertEqual([], self.target.opened_ports()) - self.check_output.return_value = '1/tcp\n2/tcp\n3/udp\n4/tcp\n5/udp\n' + self.check_output.return_value = b'1/tcp\n2/tcp\n3/udp\n4/tcp\n5/udp\n' self.assertEqual(['1', '2', '4'], self.target.opened_ports()) self.assertEqual(['1', '2', '4'], self.target.opened_ports(protocol='TCP'))