Enable Memcache for API svc token caching

On deployments of API charms using OpenStack release Mitaka or later
enable memcache for token caching.

With the release of 4.2.0 of keystonemiddleware using the
in-process token cache is no longer recommended. It is recommended
that a memcache backend to store tokens is used instead,

This installs and configures memcache for API charms
to use memcache for token caching.

http://docs.openstack.org/releasenotes/keystonemiddleware/mitaka.html#id2

Create os_release_data module for storing config like
KNOWN_RELEASES as this is not tied to the charm module an is useful
elsewhere

Change-Id: I6790f7968d0bd0493f103aec4b0b2a46333af6b0
This commit is contained in:
Liam Young 2017-01-03 13:43:39 +00:00
parent 4e9b83d7b3
commit db1818248d
5 changed files with 291 additions and 33 deletions

View File

@ -27,7 +27,9 @@ import charmhelpers.contrib.hahelpers.cluster as ch_cluster
import charmhelpers.contrib.network.ip as ch_ip
import charmhelpers.contrib.openstack.utils as ch_utils
import charmhelpers.core.hookenv as hookenv
import charmhelpers.core.host as ch_host
import charms_openstack.ip as os_ip
import charms_openstack.os_release_data as os_release_data
ADDRESS_TYPES = os_ip.ADDRESS_MAP.keys()
@ -801,6 +803,35 @@ class APIConfigurationAdapter(ConfigurationAdapter):
eps = [ep[2] for ep in self.endpoints]
return sorted(list(set(eps)))
@property
def use_memcache(self):
release = ch_utils.get_os_codename_install_source(
self.openstack_origin)
if release not in os_release_data.KNOWN_RELEASES:
return ValueError("Unkown release {}".format(release))
return (os_release_data.KNOWN_RELEASES.index(release) >=
os_release_data.KNOWN_RELEASES.index('mitaka'))
@property
def memcache_server(self):
if ch_host.lsb_release()['DISTRIB_RELEASE'] > '14.04':
memcache_server = '::1'
else:
memcache_server = 'ip6-localhost'
return memcache_server
@property
def memcache_host(self):
return '[::1]'
@property
def memcache_port(self):
return '11211'
@property
def memcache_url(self):
return 'inet6:{}:{}'.format(self.memcache_host, self.memcache_port)
def make_default_relation_adapter(base_cls, relation, properties):
"""Create a default relation adapter using a base class, and custom

View File

@ -42,8 +42,9 @@ import charmhelpers.core.unitdata as unitdata
import charmhelpers.fetch as fetch
import charms.reactive as reactive
import charms_openstack.ip as os_ip
import charms_openstack.adapters as os_adapters
import charms_openstack.ip as os_ip
import charms_openstack.os_release_data as os_release_data
# _releases{} is a dictionary of release -> class that is instantiated
@ -63,22 +64,6 @@ _singleton = None
# This is to enable the defining code to define which release is used.
_release_selector_function = None
# List of releases that OpenStackCharm based charms know about
KNOWN_RELEASES = [
'diablo',
'essex',
'folsom',
'grizzly',
'havana',
'icehouse',
'juno',
'kilo',
'liberty',
'mitaka',
'newton',
'ocata',
]
VIP_KEY = "vip"
CIDR_KEY = "vip_cidr"
IFACE_KEY = "vip_iface"
@ -377,18 +362,20 @@ def get_charm_instance(release=None, *args, **kwargs):
cls = _releases[known_releases[-1]]
else:
# check that the release is a valid release
if release not in KNOWN_RELEASES:
if release not in os_release_data.KNOWN_RELEASES:
raise RuntimeError(
"Release {} is not a known OpenStack release?".format(release))
release_index = KNOWN_RELEASES.index(release)
if release_index < KNOWN_RELEASES.index(known_releases[0]):
release_index = os_release_data.KNOWN_RELEASES.index(release)
if (release_index <
os_release_data.KNOWN_RELEASES.index(known_releases[0])):
raise RuntimeError(
"Release {} is not supported by this charm. Earliest support "
"is {} release".format(release, known_releases[0]))
else:
# try to find the release that is supported.
for known_release in reversed(known_releases):
if release_index >= KNOWN_RELEASES.index(known_release):
if (release_index >=
os_release_data.KNOWN_RELEASES.index(known_release)):
cls = _releases[known_release]
break
if cls is None:
@ -458,7 +445,7 @@ class OpenStackCharmMeta(type):
return
if 'release' in members.keys():
release = members['release']
if release not in KNOWN_RELEASES:
if release not in os_release_data.KNOWN_RELEASES:
raise RuntimeError(
"Release {} is not a known OpenStack release"
.format(release))
@ -559,6 +546,7 @@ class OpenStackCharm(object):
ha_resources = []
adapters_class = None
HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
MEMCACHE_CONF = '/etc/memcached.conf'
package_codenames = {}
@property
@ -728,7 +716,9 @@ class OpenStackCharm(object):
protocol = protocol.lower()
else:
protocol = ''
lines = [l for l in subprocess.check_output(_args).split() if l]
lines = [l for l in
subprocess.check_output(_args).decode('UTF-8').split()
if l]
ports = []
for line in lines:
p, p_type = line.split('/')
@ -1245,6 +1235,11 @@ class OpenStackAPICharm(OpenStackCharm):
# If None, then the default ConfigurationAdapter is used.
configuration_class = os_adapters.APIConfigurationAdapter
def upgrade_charm(self):
"""Setup token cache in case previous charm version did not."""
self.setup_token_cache()
super(OpenStackAPICharm, self).upgrade_charm()
def install(self):
"""Install packages related to this charm based on
contents of self.packages attribute.
@ -1252,6 +1247,36 @@ class OpenStackAPICharm(OpenStackCharm):
self.configure_source()
super(OpenStackAPICharm, self).install()
def setup_token_cache(self):
"""Check if a token cache package is needed and install it if it is"""
if fetch.filter_installed_packages(self.token_cache_pkgs()):
self.install()
def enable_memcache(self, release=None):
"""Determine if memcache should be enabled on the local unit
@param release: release of OpenStack currently deployed
@returns boolean Whether memcache should be enabled
"""
if not release:
release = os_utils.get_os_codename_install_source(
self.config['openstack-origin'])
if release not in os_release_data.KNOWN_RELEASES:
return ValueError("Unkown release {}".format(release))
return (os_release_data.KNOWN_RELEASES.index(release) >=
os_release_data.KNOWN_RELEASES.index('mitaka'))
def token_cache_pkgs(self, release=None):
"""Determine additional packages needed for token caching
@param release: release of OpenStack currently deployed
@returns List of package to enable token caching
"""
packages = []
if self.enable_memcache(release=release):
packages.extend(['memcached', 'python-memcache'])
return packages
def get_amqp_credentials(self):
"""Provide the default amqp username and vhost as a tuple.
@ -1290,6 +1315,30 @@ class OpenStackAPICharm(OpenStackCharm):
"get_database_setup() needs to be overriden in the derived "
"class")
@property
def all_packages(self):
"""List of packages to be installed
@return ['pkg1', 'pkg2', ...]
"""
return (super(OpenStackAPICharm, self).all_packages +
self.token_cache_pkgs())
@property
def full_restart_map(self):
"""Map of services to be restarted if a file changes
@return {
'file1': ['svc1', 'svc3'],
'file2': ['svc2', 'svc3'],
...
}
"""
_restart_map = super(OpenStackAPICharm, self).full_restart_map.copy()
if self.enable_memcache():
_restart_map[self.MEMCACHE_CONF] = ['memcached']
return _restart_map
class HAOpenStackCharm(OpenStackAPICharm):
@ -1332,7 +1381,7 @@ class HAOpenStackCharm(OpenStackAPICharm):
@return ['pkg1', 'pkg2', ...]
"""
_packages = self.packages[:]
_packages = super(HAOpenStackCharm, self).all_packages
if self.haproxy_enabled():
_packages.append('haproxy')
if self.apache_enabled():
@ -1349,7 +1398,7 @@ class HAOpenStackCharm(OpenStackAPICharm):
...
}
"""
_restart_map = self.restart_map.copy()
_restart_map = super(HAOpenStackCharm, self).full_restart_map
if self.haproxy_enabled():
_restart_map[self.HAPROXY_CONF] = ['haproxy']
if self.apache_enabled():

View File

@ -0,0 +1,33 @@
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# OpenStackCharm() - base class for build OpenStack charms from for the
# reactive framework.
# need/want absolute imports for the package imports to work properly
KNOWN_RELEASES = [
'diablo',
'essex',
'folsom',
'grizzly',
'havana',
'icehouse',
'juno',
'kilo',
'liberty',
'mitaka',
'newton',
'ocata',
]

View File

@ -698,6 +698,43 @@ class TestAPIConfigurationAdapter(unittest.TestCase):
c = adapters.APIConfigurationAdapter()
self.assertEqual(c.determine_service_port(80), 70)
def test_use_memcache(self):
test_config = {'openstack-origin': 'distro'}
with mock.patch.object(adapters.hookenv, 'config',
new=lambda: test_config):
with mock.patch.object(adapters.ch_utils,
'get_os_codename_install_source',
return_value='liberty'):
c = adapters.APIConfigurationAdapter()
self.assertFalse(c.use_memcache)
with mock.patch.object(adapters.ch_utils,
'get_os_codename_install_source',
return_value='newton'):
c = adapters.APIConfigurationAdapter()
self.assertTrue(c.use_memcache)
def test_memcache_server(self):
with mock.patch.object(adapters.ch_host, 'lsb_release',
return_value={'DISTRIB_RELEASE': '14.04'}):
c = adapters.APIConfigurationAdapter()
self.assertEqual(c.memcache_server, 'ip6-localhost')
with mock.patch.object(adapters.ch_host, 'lsb_release',
return_value={'DISTRIB_RELEASE': '16.04'}):
c = adapters.APIConfigurationAdapter()
self.assertEqual(c.memcache_server, '::1')
def test_memcache_host(self):
self.assertEqual(adapters.APIConfigurationAdapter().memcache_host,
'[::1]')
def test_memcache_port(self):
self.assertEqual(adapters.APIConfigurationAdapter().memcache_port,
'11211')
def test_memcache_url(self):
self.assertEqual(adapters.APIConfigurationAdapter().memcache_url,
'inet6:[::1]:11211')
class FakePeerHARelationAdapter(object):

View File

@ -495,7 +495,7 @@ class TestOpenStackCharm(BaseOpenStackCharmTest):
'filter_installed_packages',
name='fip',
return_value=None)
self.patch_object(chm.subprocess, 'check_output', return_value='\n')
self.patch_object(chm.subprocess, 'check_output', return_value=b'\n')
self.target.install()
self.target.set_state.assert_called_once_with('charmname-installed')
self.fip.assert_called_once_with([])
@ -634,20 +634,61 @@ class TestOpenStackAPICharm(BaseOpenStackCharmTest):
super(TestOpenStackAPICharm, self).setUp(chm.OpenStackAPICharm,
TEST_CONFIG)
def test_upgrade_charm(self):
self.patch_target('setup_token_cache')
self.patch_target('update_api_ports')
self.target.upgrade_charm()
self.target.setup_token_cache.assert_called_once_with()
def test_install(self):
# Test set_state and configure_source are called
self.patch_target('set_state')
self.patch_target('configure_source')
self.patch_target('enable_memcache', return_value=False)
self.patch_object(chm.charmhelpers.fetch,
'filter_installed_packages',
name='fip',
return_value=None)
self.patch_object(chm.subprocess, 'check_output', return_value='\n')
self.patch_object(chm.subprocess, 'check_output', return_value=b'\n')
self.target.install()
# self.target.set_state.assert_called_once_with('charmname-installed')
self.target.configure_source.assert_called_once_with()
self.fip.assert_called_once_with([])
def test_setup_token_cache(self):
self.patch_target('token_cache_pkgs')
self.patch_target('install')
self.patch_object(chm.charmhelpers.fetch,
'filter_installed_packages',
name='fip',
return_value=['memcached'])
self.target.setup_token_cache()
self.install.assert_called_once_with()
self.fip.return_value = []
self.install.reset_mock()
self.target.setup_token_cache()
self.assertFalse(self.install.called)
def test_enable_memcache(self):
self.assertFalse(self.target.enable_memcache(release='liberty'))
self.assertTrue(self.target.enable_memcache(release='newton'))
self.patch_target('config', new={'openstack-origin': 'distro'})
self.patch_object(chm.os_utils,
'get_os_codename_install_source',
name='gocis')
self.gocis.return_value = 'liberty'
self.assertFalse(self.target.enable_memcache())
self.gocis.return_value = 'newton'
self.assertTrue(self.target.enable_memcache())
def test_token_cache_pkgs(self):
self.patch_target('enable_memcache')
self.enable_memcache.return_value = True
self.assertEqual(self.target.token_cache_pkgs(), ['memcached',
'python-memcache'])
self.enable_memcache.return_value = False
self.assertEqual(self.target.token_cache_pkgs(), [])
def test_get_amqp_credentials(self):
# verify that the instance throws an error if not overriden
with self.assertRaises(RuntimeError):
@ -658,6 +699,29 @@ class TestOpenStackAPICharm(BaseOpenStackCharmTest):
with self.assertRaises(RuntimeError):
self.target.get_database_setup()
def test_all_packages(self):
self.patch_target('enable_memcache')
self.patch_target('packages', new=['pkg1', 'pkg2'])
self.enable_memcache.return_value = True
self.assertEqual(self.target.all_packages,
['pkg1', 'pkg2', 'memcached', 'python-memcache'])
self.enable_memcache.return_value = False
self.assertEqual(self.target.all_packages, ['pkg1', 'pkg2'])
def test_full_restart_map(self):
self.patch_target('enable_memcache')
base_restart_map = {
'conf1': ['svc1'],
'conf2': ['svc1']}
self.patch_target('restart_map', new=base_restart_map)
self.enable_memcache.return_value = True
self.assertEqual(self.target.full_restart_map,
{'conf1': ['svc1'],
'conf2': ['svc1'],
'/etc/memcached.conf': ['memcached']})
self.enable_memcache.return_value = False
self.assertEqual(self.target.full_restart_map, base_restart_map)
class TestHAOpenStackCharm(BaseOpenStackCharmTest):
# Note that this only tests the OpenStackCharm() class, which has not very
@ -668,6 +732,45 @@ class TestHAOpenStackCharm(BaseOpenStackCharmTest):
super(TestHAOpenStackCharm, self).setUp(chm.HAOpenStackCharm,
TEST_CONFIG)
def test_all_packages(self):
self.patch_target('packages', new=['pkg1'])
self.patch_target('token_cache_pkgs', return_value=[])
self.patch_target('haproxy_enabled', return_value=False)
self.patch_target('apache_enabled', return_value=False)
self.assertEqual(['pkg1'], self.target.all_packages)
self.token_cache_pkgs.return_value = ['memcache']
self.haproxy_enabled.return_value = True
self.apache_enabled.return_value = True
self.assertEqual(['pkg1', 'memcache', 'haproxy', 'apache2'],
self.target.all_packages)
def test_full_restart_map_disabled(self):
base_restart_map = {
'conf1': ['svc1'],
'conf2': ['svc1']}
self.patch_target('restart_map', new=base_restart_map)
self.patch_target('enable_memcache', return_value=False)
self.patch_target('haproxy_enabled', return_value=False)
self.patch_target('apache_enabled', return_value=False)
self.assertEqual(base_restart_map, self.target.full_restart_map)
def test_full_restart_map_enabled(self):
base_restart_map = {
'conf1': ['svc1'],
'conf2': ['svc1']}
self.patch_target('restart_map', new=base_restart_map)
self.patch_target('enable_memcache', return_value=True)
self.patch_target('haproxy_enabled', return_value=True)
self.patch_target('apache_enabled', return_value=True)
self.assertEqual(
self.target.full_restart_map,
{'/etc/apache2/sites-available/openstack_https_frontend.conf':
['apache2'],
'/etc/haproxy/haproxy.cfg': ['haproxy'],
'/etc/memcached.conf': ['memcached'],
'conf1': ['svc1'],
'conf2': ['svc1']})
def test_haproxy_enabled(self):
self.patch_target('ha_resources', new=['haproxy'])
self.assertTrue(self.target.haproxy_enabled())
@ -744,13 +847,18 @@ class TestHAOpenStackCharm(BaseOpenStackCharmTest):
self.set_state.assert_called_once_with('haproxy.stat.password',
mock.ANY)
def test_hacharm_all_packages(self):
def test_hacharm_all_packages_enabled(self):
self.patch_target('enable_memcache', return_value=False)
self.patch_target('haproxy_enabled', return_value=True)
self.assertTrue('haproxy' in self.target.all_packages)
def test_hacharm_all_packages_disabled(self):
self.patch_target('enable_memcache', return_value=False)
self.patch_target('haproxy_enabled', return_value=False)
self.assertFalse('haproxy' in self.target.all_packages)
def test_hacharm_full_restart_map(self):
self.patch_target('enable_memcache', return_value=False)
self.patch_target('haproxy_enabled', return_value=True)
self.assertTrue(
self.target.full_restart_map.get(
@ -1093,7 +1201,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest):
self.fip.side_effect = lambda x: ['p1', 'p2']
self.patch_object(chm.hookenv, 'status_set')
self.patch_object(chm.hookenv, 'apt_install')
self.patch_object(chm.subprocess, 'check_output', return_value='\n')
self.patch_object(chm.subprocess, 'check_output', return_value=b'\n')
self.target.install()
# TODO: remove next commented line as we don't set this state anymore
# self.target.set_state.assert_called_once_with('my-charm-installed')
@ -1115,7 +1223,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest):
def test_update_api_ports(self):
self.patch_object(chm.hookenv, 'open_port')
self.patch_object(chm.hookenv, 'close_port')
self.patch_object(chm.subprocess, 'check_output', return_value='\n')
self.patch_object(chm.subprocess, 'check_output', return_value=b'\n')
self.target.api_ports = {
'api': {
'public': 1,
@ -1136,7 +1244,7 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest):
# that should be closed
self.open_port.reset_mock()
self.close_port.reset_mock()
self.check_output.return_value = "1/tcp\n2/tcp\n3/udp\n4/tcp\n"
self.check_output.return_value = b"1/tcp\n2/tcp\n3/udp\n4/tcp\n"
# port 3 should be opened, port 4 should be closed.
open_calls = [mock.call(3)]
close_calls = [mock.call(4)]
@ -1146,9 +1254,9 @@ class TestMyOpenStackCharm(BaseOpenStackCharmTest):
def test_opened_ports(self):
self.patch_object(chm.subprocess, 'check_output')
self.check_output.return_value = '\n'
self.check_output.return_value = b'\n'
self.assertEqual([], self.target.opened_ports())
self.check_output.return_value = '1/tcp\n2/tcp\n3/udp\n4/tcp\n5/udp\n'
self.check_output.return_value = b'1/tcp\n2/tcp\n3/udp\n4/tcp\n5/udp\n'
self.assertEqual(['1', '2', '4'], self.target.opened_ports())
self.assertEqual(['1', '2', '4'],
self.target.opened_ports(protocol='TCP'))