Merge "Add memcache backend"

charmhelpers/contrib/openstack/ha/utils.py

@@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params,
             continue
         m = re.search('os-(.+?)-hostname', setting)
         if m:
-            networkspace = m.group(1)
+            endpoint_type = m.group(1)
+            # resolve_address's ADDRESS_MAP uses 'int' not 'internal'
+            if endpoint_type == 'internal':
+                endpoint_type = 'int'
         else:
             msg = ('Unexpected DNS hostname setting: {}. '
-                   'Cannot determine network space name'
+                   'Cannot determine endpoint_type name'
                    ''.format(setting))
             status_set('blocked', msg)
             raise DNSHAException(msg)
 
-        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
+        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
         if hostname_key in hostname_group:
             log('DNS HA: Resource {}: {} already exists in '
                 'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params,
         resources[hostname_key] = crm_ocf
         resource_params[hostname_key] = (
             'params fqdn="{}" ip_address="{}" '
-            ''.format(hostname, resolve_address(endpoint_type=networkspace,
+            ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
                                                 override=False)))
 
     if len(hostname_group) >= 1:

charmhelpers/contrib/openstack/neutron.py

@@ -59,18 +59,13 @@ def determine_dkms_package():
 
 
 def quantum_plugins():
-    from charmhelpers.contrib.openstack import context
     return {
         'ovs': {
             'config': '/etc/quantum/plugins/openvswitch/'
                       'ovs_quantum_plugin.ini',
             'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
                       'OVSQuantumPluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=QUANTUM_CONF_DIR)],
             'services': ['quantum-plugin-openvswitch-agent'],
             'packages': [determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
@@ -82,11 +77,7 @@ def quantum_plugins():
             'config': '/etc/quantum/plugins/nicira/nvp.ini',
             'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
                       'QuantumPlugin.NvpPluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=QUANTUM_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['quantum-server',
@@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron'
 
 
 def neutron_plugins():
-    from charmhelpers.contrib.openstack import context
     release = os_release('nova-common')
     plugins = {
         'ovs': {
@@ -108,11 +98,7 @@ def neutron_plugins():
                       'ovs_neutron_plugin.ini',
             'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
                       'OVSNeutronPluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': ['neutron-plugin-openvswitch-agent'],
             'packages': [determine_dkms_package(),
                          ['neutron-plugin-openvswitch-agent']],
@@ -124,11 +110,7 @@ def neutron_plugins():
             'config': '/etc/neutron/plugins/nicira/nvp.ini',
             'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
                       'NeutronPlugin.NvpPluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server',
@@ -138,11 +120,7 @@ def neutron_plugins():
         'nsx': {
             'config': '/etc/neutron/plugins/vmware/nsx.ini',
             'driver': 'vmware',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server',
@@ -152,11 +130,7 @@ def neutron_plugins():
         'n1kv': {
             'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini',
             'driver': 'neutron.plugins.cisco.network_plugin.PluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [determine_dkms_package(),
                          ['neutron-plugin-cisco']],
@@ -167,11 +141,7 @@ def neutron_plugins():
         'Calico': {
             'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
             'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': ['calico-felix',
                          'bird',
                          'neutron-dhcp-agent',
@@ -189,11 +159,7 @@ def neutron_plugins():
         'vsp': {
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
             'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
@@ -203,10 +169,7 @@ def neutron_plugins():
             'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
             'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
                        '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('database-user'),
-                                        database=config('database'),
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': ['plumgrid-lxc',
                          'iovisor-dkms'],
@@ -217,11 +180,7 @@ def neutron_plugins():
         'midonet': {
             'config': '/etc/neutron/plugins/midonet/midonet.ini',
             'driver': 'midonet.neutron.plugin.MidonetPluginV2',
-            'contexts': [
+            'contexts': [],
-                context.SharedDBContext(user=config('neutron-database-user'),
-                                        database=config('neutron-database'),
-                                        relation_prefix='neutron',
-                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [determine_dkms_package()],
             'server_packages': ['neutron-server',

charmhelpers/contrib/openstack/templates/section-oslo-cache

@@ -0,0 +1,6 @@
+[cache]
+{% if memcache_url %}
+enabled = true
+backend = oslo_cache.memcache_pool
+memcache_servers = {{ memcache_url }}
+{% endif %}

charmhelpers/core/hookenv.py

@@ -1093,6 +1093,24 @@ def network_get_primary_address(binding):
     return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+    """
+    Retrieve the network details for a relation endpoint
+
+    :param endpoint: string. The name of a relation endpoint
+    :param relation_id: int. The ID of the relation for the current context.
+    :return: dict. The loaded YAML output of the network-get query.
+    :raise: NotImplementedError if run on Juju < 2.0
+    """
+    cmd = ['network-get', endpoint, '--format', 'yaml']
+    if relation_id:
+        cmd.append('-r')
+        cmd.append(relation_id)
+    response = subprocess.check_output(cmd).decode('UTF-8').strip()
+    return yaml.safe_load(response)
+
+
 def add_metric(*args, **kwargs):
     """Add metric values. Values may be expressed with keyword arguments. For
     metric names containing dashes, these may be expressed as one or more

hooks/keystone_hooks.py

@@ -74,6 +74,7 @@ from charmhelpers.contrib.openstack.utils import (
     snap_install_requested,
     install_os_snaps,
     get_snaps_install_info_from_origin,
+    enable_memcache,
 )
 
 from keystone_utils import (
@@ -250,7 +251,8 @@ def config_changed_postupgrade():
     ensure_ssl_dirs()
 
     save_script_rc()
-    if run_in_apache():
+    release = os_release('keystone')
+    if run_in_apache(release=release):
         # Need to ensure mod_wsgi is installed and apache2 is reloaded
         # immediatly as charm querys its local keystone before restart
         # decorator can fire
@@ -265,6 +267,11 @@ def config_changed_postupgrade():
         if not is_unit_paused_set():
             restart_pid_check('apache2')
 
+    if enable_memcache(release=release):
+        # If charm or OpenStack have been upgraded then the list of required
+        # packages may have changed so ensure they are installed.
+        apt_install(filter_installed_packages(determine_packages()))
+
     configure_https()
     open_port(config('service-port'))
 

hooks/keystone_utils.py

@@ -79,6 +79,7 @@ from charmhelpers.contrib.openstack.utils import (
     snap_install_requested,
     install_os_snaps,
     get_snaps_install_info_from_origin,
+    enable_memcache,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -256,6 +257,7 @@ else:
 HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
 APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
 APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf'
+MEMCACHED_CONF = '/etc/memcached.conf'
 
 SSL_CA_NAME = 'Ubuntu Cloud'
 CLUSTER_RES = 'grp_ks_vips'
@@ -280,7 +282,8 @@ BASE_RESOURCE_MAP = OrderedDict([
                      context.SyslogContext(),
                      keystone_context.HAProxyContext(),
                      context.BindHostContext(),
-                     context.WorkerConfigContext()],
+                     context.WorkerConfigContext(),
+                     context.MemcacheContext(package='keystone')],
     }),
     (KEYSTONE_LOGGER_CONF, {
         'contexts': [keystone_context.KeystoneLoggingContext()],
@@ -507,7 +510,8 @@ def resource_map():
     """
     resource_map = deepcopy(BASE_RESOURCE_MAP)
 
-    if CompareOpenStackReleases(os_release('keystone')) < 'liberty':
+    release = os_release('keystone')
+    if CompareOpenStackReleases(release) < 'liberty':
         resource_map.pop(POLICY_JSON)
     if os.path.exists('/etc/apache2/conf-available'):
         resource_map.pop(APACHE_CONF)
@@ -556,6 +560,12 @@ def resource_map():
                     keystone_context.KeystoneContext()],
                 'services': ['apache2']
             }
+
+    if enable_memcache(release=release):
+        resource_map[MEMCACHED_CONF] = {
+            'contexts': [context.MemcacheContext()],
+            'services': ['memcached']}
+
     return resource_map
 
 
@@ -588,11 +598,12 @@ def restart_function_map():
     return rfunc_map
 
 
-def run_in_apache():
+def run_in_apache(release=None):
     """Return true if keystone API is run under apache2 with mod_wsgi in
     this release.
     """
-    return (CompareOpenStackReleases(os_release('keystone')) >= 'liberty' and
+    release = release or os_release('keystone')
+    return (CompareOpenStackReleases(release) >= 'liberty' and
             not snap_install_requested())
 
 
@@ -647,7 +658,10 @@ def api_port(service):
 def determine_packages():
     # currently all packages match service names
     if snap_install_requested():
-        return sorted(BASE_PACKAGES_SNAP)
+        pkgs = deepcopy(BASE_PACKAGES_SNAP)
+        if enable_memcache(release=os_release('keystone')):
+            pkgs = pkgs + ['memcached']
+        return sorted(pkgs)
     else:
         packages = set(services()).union(BASE_PACKAGES)
         if git_install_requested():

templates/mitaka/keystone.conf

@@ -56,7 +56,7 @@ expiration = {{ token_expiration }}
 
 {% include "parts/section-signing" %}
 
-[cache]
+{% include "section-oslo-cache" %}
 
 [policy]
 driver = sql

templates/ocata/keystone.conf

@@ -56,7 +56,7 @@ expiration = {{ token_expiration }}
 
 {% include "parts/section-signing" %}
 
-[cache]
+{% include "section-oslo-cache" %}
 
 [policy]
 driver = sql

tests/charmhelpers/core/hookenv.py

@@ -1093,6 +1093,24 @@ def network_get_primary_address(binding):
     return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def network_get(endpoint, relation_id=None):
+    """
+    Retrieve the network details for a relation endpoint
+
+    :param endpoint: string. The name of a relation endpoint
+    :param relation_id: int. The ID of the relation for the current context.
+    :return: dict. The loaded YAML output of the network-get query.
+    :raise: NotImplementedError if run on Juju < 2.0
+    """
+    cmd = ['network-get', endpoint, '--format', 'yaml']
+    if relation_id:
+        cmd.append('-r')
+        cmd.append(relation_id)
+    response = subprocess.check_output(cmd).decode('UTF-8').strip()
+    return yaml.safe_load(response)
+
+
 def add_metric(*args, **kwargs):
     """Add metric values. Values may be expressed with keyword arguments. For
     metric names containing dashes, these may be expressed as one or more

unit_tests/test_keystone_hooks.py

@@ -82,6 +82,7 @@ TO_PATCH = [
     'is_elected_leader',
     'get_hacluster_config',
     'is_clustered',
+    'enable_memcache',
     # keystone_utils
     'restart_map',
     'register_configs',
@@ -384,6 +385,7 @@ class KeystoneRelationTests(CharmTestCase):
                     'identity-service': ['identity-service:0']}
             return rids.get(relation, [])
 
+        self.enable_memcache.return_value = False
         self.os_release.return_value = 'mitaka'
         self.relation_ids.side_effect = fake_relation_ids
 
@@ -452,6 +454,7 @@ class KeystoneRelationTests(CharmTestCase):
             rids = {}
             return rids.get(relation, [])
 
+        self.enable_memcache.return_value = False
         self.os_release.return_value = 'mitaka'
         self.relation_ids.side_effect = fake_relation_ids
 
@@ -522,6 +525,7 @@ class KeystoneRelationTests(CharmTestCase):
             return rids.get(relation, [])
 
         self.os_release.return_value = 'mitaka'
+        self.enable_memcache.return_value = False
         self.relation_ids.side_effect = fake_relation_ids
 
         mock_run_in_apache.return_value = False
@@ -585,6 +589,7 @@ class KeystoneRelationTests(CharmTestCase):
                                         mock_run_in_apache,
                                         update,
                                         mock_update_domains):
+        self.enable_memcache.return_value = False
         mock_run_in_apache.return_value = False
         git_requested.return_value = True
         mock_ensure_ssl_cert_master.return_value = False
@@ -640,6 +645,7 @@ class KeystoneRelationTests(CharmTestCase):
                                                           mock_db_init,
                                                           mock_initialise_pki,
                                                           mock_run_in_apache):
+        self.enable_memcache.return_value = False
         mock_run_in_apache.return_value = False
         ensure_ssl_cert.return_value = False
         peer_units.return_value = []

unit_tests/test_keystone_utils.py

@@ -134,6 +134,30 @@ class TestKeystoneUtils(CharmTestCase):
         ]
         self.assertEqual(fake_renderer.register.call_args_list, ex_reg)
 
+    @patch.object(utils, 'git_determine_usr_bin')
+    @patch.object(utils, 'snap_install_requested')
+    @patch.object(utils, 'os')
+    def test_resource_map_enable_memcache_mitaka(self, mock_os,
+                                                 snap_install_requested,
+                                                 git_determine_usr_bin):
+        self.os_release.return_value = 'mitaka'
+        snap_install_requested.return_value = False
+        git_determine_usr_bin.return_value = '/usr/bin'
+        mock_os.path.exists.return_value = True
+        self.assertTrue('/etc/memcached.conf' in utils.resource_map().keys())
+
+    @patch.object(utils, 'git_determine_usr_bin')
+    @patch.object(utils, 'snap_install_requested')
+    @patch.object(utils, 'os')
+    def test_resource_map_enable_memcache_liberty(self, mock_os,
+                                                  snap_install_requested,
+                                                  git_determine_usr_bin):
+        self.os_release.return_value = 'liberty'
+        snap_install_requested.return_value = False
+        git_determine_usr_bin.return_value = '/usr/bin'
+        mock_os.path.exists.return_value = True
+        self.assertFalse('/etc/memcached.conf' in utils.resource_map().keys())
+
     def test_determine_ports(self):
         self.test_config.set('admin-port', '80')
         self.test_config.set('service-port', '81')
@@ -149,6 +173,17 @@ class TestKeystoneUtils(CharmTestCase):
         ex = utils.BASE_PACKAGES + ['keystone', 'python-keystoneclient']
         self.assertEqual(set(ex), set(result))
 
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_determine_packages_mitaka(self, _config):
+        self.os_release.return_value = 'mitaka'
+        self.snap_install_requested.return_value = False
+        _config.return_value = None
+        result = utils.determine_packages()
+        ex = utils.BASE_PACKAGES + [
+            'keystone', 'python-keystoneclient', 'libapache2-mod-wsgi',
+            'memcached']
+        self.assertEqual(set(ex), set(result))
+
     @patch('charmhelpers.contrib.openstack.utils.config')
     def test_determine_packages_git(self, _config):
         self.os_release.return_value = 'havana'
@@ -159,6 +194,15 @@ class TestKeystoneUtils(CharmTestCase):
             ex.remove(p)
         self.assertEqual(set(ex), set(result))
 
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_determine_packages_snap_install(self, _config):
+        self.os_release.return_value = 'mitaka'
+        self.snap_install_requested.return_value = True
+        _config.return_value = None
+        result = utils.determine_packages()
+        ex = utils.BASE_PACKAGES_SNAP + ['memcached']
+        self.assertEqual(set(ex), set(result))
+
     @patch.object(utils, 'disable_unused_apache_sites')
     @patch('os.path.exists')
     @patch.object(utils, 'run_in_apache')
@@ -1301,3 +1345,15 @@ class TestKeystoneUtils(CharmTestCase):
             1, 'a2dissite')
         utils.disable_unused_apache_sites()
         os_remove.assert_called_with(utils.UNUSED_APACHE_SITE_FILES[0])
+
+    def test_run_in_apache_kilo(self):
+        self.os_release.return_value = 'kilo'
+        self.assertFalse(utils.run_in_apache())
+
+    def test_run_in_apache_liberty(self):
+        self.os_release.return_value = 'liberty'
+        self.assertTrue(utils.run_in_apache())
+
+    def test_run_in_apache_set_release(self):
+        self.os_release.return_value = 'kilo'
+        self.assertTrue(utils.run_in_apache(release='liberty'))