diff --git a/charmhelpers/contrib/openstack/ha/utils.py b/charmhelpers/contrib/openstack/ha/utils.py index 254a90e7..9a4d79c1 100644 --- a/charmhelpers/contrib/openstack/ha/utils.py +++ b/charmhelpers/contrib/openstack/ha/utils.py @@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params, continue m = re.search('os-(.+?)-hostname', setting) if m: - networkspace = m.group(1) + endpoint_type = m.group(1) + # resolve_address's ADDRESS_MAP uses 'int' not 'internal' + if endpoint_type == 'internal': + endpoint_type = 'int' else: msg = ('Unexpected DNS hostname setting: {}. ' - 'Cannot determine network space name' + 'Cannot determine endpoint_type name' ''.format(setting)) status_set('blocked', msg) raise DNSHAException(msg) - hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace) + hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type) if hostname_key in hostname_group: log('DNS HA: Resource {}: {} already exists in ' 'hostname group - skipping'.format(hostname_key, hostname), @@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params, resources[hostname_key] = crm_ocf resource_params[hostname_key] = ( 'params fqdn="{}" ip_address="{}" ' - ''.format(hostname, resolve_address(endpoint_type=networkspace, + ''.format(hostname, resolve_address(endpoint_type=endpoint_type, override=False))) if len(hostname_group) >= 1: diff --git a/charmhelpers/contrib/openstack/neutron.py b/charmhelpers/contrib/openstack/neutron.py index 37fa0eb0..0f847f56 100644 --- a/charmhelpers/contrib/openstack/neutron.py +++ b/charmhelpers/contrib/openstack/neutron.py @@ -59,18 +59,13 @@ def determine_dkms_package(): def quantum_plugins(): - from charmhelpers.contrib.openstack import context return { 'ovs': { 'config': '/etc/quantum/plugins/openvswitch/' 'ovs_quantum_plugin.ini', 'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.' 'OVSQuantumPluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=QUANTUM_CONF_DIR)], + 'contexts': [], 'services': ['quantum-plugin-openvswitch-agent'], 'packages': [determine_dkms_package(), ['quantum-plugin-openvswitch-agent']], @@ -82,11 +77,7 @@ def quantum_plugins(): 'config': '/etc/quantum/plugins/nicira/nvp.ini', 'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.' 'QuantumPlugin.NvpPluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=QUANTUM_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [], 'server_packages': ['quantum-server', @@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron' def neutron_plugins(): - from charmhelpers.contrib.openstack import context release = os_release('nova-common') plugins = { 'ovs': { @@ -108,11 +98,7 @@ def neutron_plugins(): 'ovs_neutron_plugin.ini', 'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.' 'OVSNeutronPluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': ['neutron-plugin-openvswitch-agent'], 'packages': [determine_dkms_package(), ['neutron-plugin-openvswitch-agent']], @@ -124,11 +110,7 @@ def neutron_plugins(): 'config': '/etc/neutron/plugins/nicira/nvp.ini', 'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.' 'NeutronPlugin.NvpPluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [], 'server_packages': ['neutron-server', @@ -138,11 +120,7 @@ def neutron_plugins(): 'nsx': { 'config': '/etc/neutron/plugins/vmware/nsx.ini', 'driver': 'vmware', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [], 'server_packages': ['neutron-server', @@ -152,11 +130,7 @@ def neutron_plugins(): 'n1kv': { 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [determine_dkms_package(), ['neutron-plugin-cisco']], @@ -167,11 +141,7 @@ def neutron_plugins(): 'Calico': { 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini', 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': ['calico-felix', 'bird', 'neutron-dhcp-agent', @@ -189,11 +159,7 @@ def neutron_plugins(): 'vsp': { 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [], 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], @@ -203,10 +169,7 @@ def neutron_plugins(): 'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini', 'driver': ('neutron.plugins.plumgrid.plumgrid_plugin' '.plumgrid_plugin.NeutronPluginPLUMgridV2'), - 'contexts': [ - context.SharedDBContext(user=config('database-user'), - database=config('database'), - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': ['plumgrid-lxc', 'iovisor-dkms'], @@ -217,11 +180,7 @@ def neutron_plugins(): 'midonet': { 'config': '/etc/neutron/plugins/midonet/midonet.ini', 'driver': 'midonet.neutron.plugin.MidonetPluginV2', - 'contexts': [ - context.SharedDBContext(user=config('neutron-database-user'), - database=config('neutron-database'), - relation_prefix='neutron', - ssl_dir=NEUTRON_CONF_DIR)], + 'contexts': [], 'services': [], 'packages': [determine_dkms_package()], 'server_packages': ['neutron-server', diff --git a/charmhelpers/contrib/openstack/templates/section-oslo-cache b/charmhelpers/contrib/openstack/templates/section-oslo-cache new file mode 100644 index 00000000..e056a32a --- /dev/null +++ b/charmhelpers/contrib/openstack/templates/section-oslo-cache @@ -0,0 +1,6 @@ +[cache] +{% if memcache_url %} +enabled = true +backend = oslo_cache.memcache_pool +memcache_servers = {{ memcache_url }} +{% endif %} diff --git a/charmhelpers/core/hookenv.py b/charmhelpers/core/hookenv.py index 622987d1..fb96f2dd 100644 --- a/charmhelpers/core/hookenv.py +++ b/charmhelpers/core/hookenv.py @@ -1093,6 +1093,24 @@ def network_get_primary_address(binding): return subprocess.check_output(cmd).decode('UTF-8').strip() +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def network_get(endpoint, relation_id=None): + """ + Retrieve the network details for a relation endpoint + + :param endpoint: string. The name of a relation endpoint + :param relation_id: int. The ID of the relation for the current context. + :return: dict. The loaded YAML output of the network-get query. + :raise: NotImplementedError if run on Juju < 2.0 + """ + cmd = ['network-get', endpoint, '--format', 'yaml'] + if relation_id: + cmd.append('-r') + cmd.append(relation_id) + response = subprocess.check_output(cmd).decode('UTF-8').strip() + return yaml.safe_load(response) + + def add_metric(*args, **kwargs): """Add metric values. Values may be expressed with keyword arguments. For metric names containing dashes, these may be expressed as one or more diff --git a/hooks/keystone_hooks.py b/hooks/keystone_hooks.py index 0e830d7b..f43c0259 100755 --- a/hooks/keystone_hooks.py +++ b/hooks/keystone_hooks.py @@ -74,6 +74,7 @@ from charmhelpers.contrib.openstack.utils import ( snap_install_requested, install_os_snaps, get_snaps_install_info_from_origin, + enable_memcache, ) from keystone_utils import ( @@ -250,7 +251,8 @@ def config_changed_postupgrade(): ensure_ssl_dirs() save_script_rc() - if run_in_apache(): + release = os_release('keystone') + if run_in_apache(release=release): # Need to ensure mod_wsgi is installed and apache2 is reloaded # immediatly as charm querys its local keystone before restart # decorator can fire @@ -265,6 +267,11 @@ def config_changed_postupgrade(): if not is_unit_paused_set(): restart_pid_check('apache2') + if enable_memcache(release=release): + # If charm or OpenStack have been upgraded then the list of required + # packages may have changed so ensure they are installed. + apt_install(filter_installed_packages(determine_packages())) + configure_https() open_port(config('service-port')) diff --git a/hooks/keystone_utils.py b/hooks/keystone_utils.py index 059378b4..ce967d51 100644 --- a/hooks/keystone_utils.py +++ b/hooks/keystone_utils.py @@ -79,6 +79,7 @@ from charmhelpers.contrib.openstack.utils import ( snap_install_requested, install_os_snaps, get_snaps_install_info_from_origin, + enable_memcache, ) from charmhelpers.contrib.python.packages import ( @@ -256,6 +257,7 @@ else: HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend' APACHE_24_CONF = '/etc/apache2/sites-available/openstack_https_frontend.conf' +MEMCACHED_CONF = '/etc/memcached.conf' SSL_CA_NAME = 'Ubuntu Cloud' CLUSTER_RES = 'grp_ks_vips' @@ -280,7 +282,8 @@ BASE_RESOURCE_MAP = OrderedDict([ context.SyslogContext(), keystone_context.HAProxyContext(), context.BindHostContext(), - context.WorkerConfigContext()], + context.WorkerConfigContext(), + context.MemcacheContext(package='keystone')], }), (KEYSTONE_LOGGER_CONF, { 'contexts': [keystone_context.KeystoneLoggingContext()], @@ -507,7 +510,8 @@ def resource_map(): """ resource_map = deepcopy(BASE_RESOURCE_MAP) - if CompareOpenStackReleases(os_release('keystone')) < 'liberty': + release = os_release('keystone') + if CompareOpenStackReleases(release) < 'liberty': resource_map.pop(POLICY_JSON) if os.path.exists('/etc/apache2/conf-available'): resource_map.pop(APACHE_CONF) @@ -556,6 +560,12 @@ def resource_map(): keystone_context.KeystoneContext()], 'services': ['apache2'] } + + if enable_memcache(release=release): + resource_map[MEMCACHED_CONF] = { + 'contexts': [context.MemcacheContext()], + 'services': ['memcached']} + return resource_map @@ -588,11 +598,12 @@ def restart_function_map(): return rfunc_map -def run_in_apache(): +def run_in_apache(release=None): """Return true if keystone API is run under apache2 with mod_wsgi in this release. """ - return (CompareOpenStackReleases(os_release('keystone')) >= 'liberty' and + release = release or os_release('keystone') + return (CompareOpenStackReleases(release) >= 'liberty' and not snap_install_requested()) @@ -647,7 +658,10 @@ def api_port(service): def determine_packages(): # currently all packages match service names if snap_install_requested(): - return sorted(BASE_PACKAGES_SNAP) + pkgs = deepcopy(BASE_PACKAGES_SNAP) + if enable_memcache(release=os_release('keystone')): + pkgs = pkgs + ['memcached'] + return sorted(pkgs) else: packages = set(services()).union(BASE_PACKAGES) if git_install_requested(): diff --git a/templates/mitaka/keystone.conf b/templates/mitaka/keystone.conf index ee6b27d2..be2f5b76 100644 --- a/templates/mitaka/keystone.conf +++ b/templates/mitaka/keystone.conf @@ -56,7 +56,7 @@ expiration = {{ token_expiration }} {% include "parts/section-signing" %} -[cache] +{% include "section-oslo-cache" %} [policy] driver = sql diff --git a/templates/ocata/keystone.conf b/templates/ocata/keystone.conf index 3fd4b401..4ce530da 100644 --- a/templates/ocata/keystone.conf +++ b/templates/ocata/keystone.conf @@ -56,7 +56,7 @@ expiration = {{ token_expiration }} {% include "parts/section-signing" %} -[cache] +{% include "section-oslo-cache" %} [policy] driver = sql diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index 622987d1..fb96f2dd 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -1093,6 +1093,24 @@ def network_get_primary_address(binding): return subprocess.check_output(cmd).decode('UTF-8').strip() +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def network_get(endpoint, relation_id=None): + """ + Retrieve the network details for a relation endpoint + + :param endpoint: string. The name of a relation endpoint + :param relation_id: int. The ID of the relation for the current context. + :return: dict. The loaded YAML output of the network-get query. + :raise: NotImplementedError if run on Juju < 2.0 + """ + cmd = ['network-get', endpoint, '--format', 'yaml'] + if relation_id: + cmd.append('-r') + cmd.append(relation_id) + response = subprocess.check_output(cmd).decode('UTF-8').strip() + return yaml.safe_load(response) + + def add_metric(*args, **kwargs): """Add metric values. Values may be expressed with keyword arguments. For metric names containing dashes, these may be expressed as one or more diff --git a/unit_tests/test_keystone_hooks.py b/unit_tests/test_keystone_hooks.py index ef1caa02..a19b94bb 100644 --- a/unit_tests/test_keystone_hooks.py +++ b/unit_tests/test_keystone_hooks.py @@ -82,6 +82,7 @@ TO_PATCH = [ 'is_elected_leader', 'get_hacluster_config', 'is_clustered', + 'enable_memcache', # keystone_utils 'restart_map', 'register_configs', @@ -384,6 +385,7 @@ class KeystoneRelationTests(CharmTestCase): 'identity-service': ['identity-service:0']} return rids.get(relation, []) + self.enable_memcache.return_value = False self.os_release.return_value = 'mitaka' self.relation_ids.side_effect = fake_relation_ids @@ -452,6 +454,7 @@ class KeystoneRelationTests(CharmTestCase): rids = {} return rids.get(relation, []) + self.enable_memcache.return_value = False self.os_release.return_value = 'mitaka' self.relation_ids.side_effect = fake_relation_ids @@ -522,6 +525,7 @@ class KeystoneRelationTests(CharmTestCase): return rids.get(relation, []) self.os_release.return_value = 'mitaka' + self.enable_memcache.return_value = False self.relation_ids.side_effect = fake_relation_ids mock_run_in_apache.return_value = False @@ -585,6 +589,7 @@ class KeystoneRelationTests(CharmTestCase): mock_run_in_apache, update, mock_update_domains): + self.enable_memcache.return_value = False mock_run_in_apache.return_value = False git_requested.return_value = True mock_ensure_ssl_cert_master.return_value = False @@ -640,6 +645,7 @@ class KeystoneRelationTests(CharmTestCase): mock_db_init, mock_initialise_pki, mock_run_in_apache): + self.enable_memcache.return_value = False mock_run_in_apache.return_value = False ensure_ssl_cert.return_value = False peer_units.return_value = [] diff --git a/unit_tests/test_keystone_utils.py b/unit_tests/test_keystone_utils.py index f14536de..ebfd5fdd 100644 --- a/unit_tests/test_keystone_utils.py +++ b/unit_tests/test_keystone_utils.py @@ -134,6 +134,30 @@ class TestKeystoneUtils(CharmTestCase): ] self.assertEqual(fake_renderer.register.call_args_list, ex_reg) + @patch.object(utils, 'git_determine_usr_bin') + @patch.object(utils, 'snap_install_requested') + @patch.object(utils, 'os') + def test_resource_map_enable_memcache_mitaka(self, mock_os, + snap_install_requested, + git_determine_usr_bin): + self.os_release.return_value = 'mitaka' + snap_install_requested.return_value = False + git_determine_usr_bin.return_value = '/usr/bin' + mock_os.path.exists.return_value = True + self.assertTrue('/etc/memcached.conf' in utils.resource_map().keys()) + + @patch.object(utils, 'git_determine_usr_bin') + @patch.object(utils, 'snap_install_requested') + @patch.object(utils, 'os') + def test_resource_map_enable_memcache_liberty(self, mock_os, + snap_install_requested, + git_determine_usr_bin): + self.os_release.return_value = 'liberty' + snap_install_requested.return_value = False + git_determine_usr_bin.return_value = '/usr/bin' + mock_os.path.exists.return_value = True + self.assertFalse('/etc/memcached.conf' in utils.resource_map().keys()) + def test_determine_ports(self): self.test_config.set('admin-port', '80') self.test_config.set('service-port', '81') @@ -149,6 +173,17 @@ class TestKeystoneUtils(CharmTestCase): ex = utils.BASE_PACKAGES + ['keystone', 'python-keystoneclient'] self.assertEqual(set(ex), set(result)) + @patch('charmhelpers.contrib.openstack.utils.config') + def test_determine_packages_mitaka(self, _config): + self.os_release.return_value = 'mitaka' + self.snap_install_requested.return_value = False + _config.return_value = None + result = utils.determine_packages() + ex = utils.BASE_PACKAGES + [ + 'keystone', 'python-keystoneclient', 'libapache2-mod-wsgi', + 'memcached'] + self.assertEqual(set(ex), set(result)) + @patch('charmhelpers.contrib.openstack.utils.config') def test_determine_packages_git(self, _config): self.os_release.return_value = 'havana' @@ -159,6 +194,15 @@ class TestKeystoneUtils(CharmTestCase): ex.remove(p) self.assertEqual(set(ex), set(result)) + @patch('charmhelpers.contrib.openstack.utils.config') + def test_determine_packages_snap_install(self, _config): + self.os_release.return_value = 'mitaka' + self.snap_install_requested.return_value = True + _config.return_value = None + result = utils.determine_packages() + ex = utils.BASE_PACKAGES_SNAP + ['memcached'] + self.assertEqual(set(ex), set(result)) + @patch.object(utils, 'disable_unused_apache_sites') @patch('os.path.exists') @patch.object(utils, 'run_in_apache') @@ -1301,3 +1345,15 @@ class TestKeystoneUtils(CharmTestCase): 1, 'a2dissite') utils.disable_unused_apache_sites() os_remove.assert_called_with(utils.UNUSED_APACHE_SITE_FILES[0]) + + def test_run_in_apache_kilo(self): + self.os_release.return_value = 'kilo' + self.assertFalse(utils.run_in_apache()) + + def test_run_in_apache_liberty(self): + self.os_release.return_value = 'liberty' + self.assertTrue(utils.run_in_apache()) + + def test_run_in_apache_set_release(self): + self.os_release.return_value = 'kilo' + self.assertTrue(utils.run_in_apache(release='liberty'))