From b355ea0473af5f1cf47c23ac5dadbeea52942bb0 Mon Sep 17 00:00:00 2001 From: Shane Peters Date: Mon, 25 Jun 2018 12:07:49 -0400 Subject: [PATCH] Add functionality for vendor_data Using vendor metadata helps alleviate the need to spin custom images for things like package mirrors, timezones, or network proxies. Adds new config option 'vendor-data' which takes a JSON formated string to be used as static vendor metadata. Adds new config option 'vendor-data-url' which takes a URL which serves dynamic JSON formatted vendor metadata. Adds new NovaMetadataContext class which writes /etc/nova/vendor_data.json and enables it via nova.conf. Closes-Bug: 1777714 Change-Id: I1d70804e59d42b0651a462c81e01d9c95626f27d --- config.yaml | 16 +++++++++ hooks/neutron_contexts.py | 31 ++++++++++++++++- hooks/neutron_hooks.py | 4 +++ hooks/neutron_utils.py | 21 +++++++++++- templates/mitaka/nova.conf | 5 +++ templates/newton/nova.conf | 48 ++++++++++++++++++++++++++ unit_tests/test_neutron_contexts.py | 52 +++++++++++++++++++++++++++++ unit_tests/test_neutron_utils.py | 19 +++++++++++ 8 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 templates/newton/nova.conf diff --git a/config.yaml b/config.yaml index 22b627b5..cf656c0c 100644 --- a/config.yaml +++ b/config.yaml @@ -251,3 +251,19 @@ options: description: | IPFIX target wit the format "IP_Address:Port". This will enable IPFIX exporting on all OVS bridges to the target, including br-int and br-ext. + vendor-data: + type: string + default: + description: | + A JSON-formatted string that will serve as vendor metadata + (via "StaticJSON" provider) to all VM's within an OpenStack deployment, + regardless of project or domain. + vendor-data-url: + type: string + default: + description: | + A URL serving JSON-formatted data that will serve as vendor metadata + (via "DynamicJSON" provider) to all VM's within an OpenStack deployment, + regardless of project or domain. + . + Only supported in OpenStack Newton and higher. diff --git a/hooks/neutron_contexts.py b/hooks/neutron_contexts.py index 17eee2fe..f4f32c98 100644 --- a/hooks/neutron_contexts.py +++ b/hooks/neutron_contexts.py @@ -2,6 +2,7 @@ import os import uuid from charmhelpers.core.hookenv import ( + log, ERROR, config, unit_get, network_get_primary_address, @@ -11,7 +12,11 @@ from charmhelpers.contrib.openstack.context import ( NeutronAPIContext, config_flags_parser, ) -from charmhelpers.contrib.hahelpers.cluster import( +from charmhelpers.contrib.openstack.utils import ( + os_release, + CompareOpenStackReleases, +) +from charmhelpers.contrib.hahelpers.cluster import ( eligible_leader ) from charmhelpers.contrib.network.ip import ( @@ -145,6 +150,30 @@ class NeutronGatewayContext(NeutronAPIContext): return ctxt +class NovaMetadataContext(OSContextGenerator): + + def __call__(self): + ctxt = {} + ctxt['vendordata_providers'] = [] + vdata = config('vendor-data') + vdata_url = config('vendor-data-url') + cmp_os_release = CompareOpenStackReleases(os_release('neutron-common')) + + if vdata: + ctxt['vendor_data'] = True + ctxt['vendordata_providers'].append('StaticJSON') + + if vdata_url: + if cmp_os_release > 'mitaka': + ctxt['vendor_data_url'] = vdata_url + ctxt['vendordata_providers'].append('DynamicJSON') + else: + log('Dynamic vendor data unsupported' + ' for {}.'.format(cmp_os_release), level=ERROR) + + return ctxt + + SHARED_SECRET = "/etc/{}/secret.txt" diff --git a/hooks/neutron_hooks.py b/hooks/neutron_hooks.py index 7925be0f..938b7a5f 100755 --- a/hooks/neutron_hooks.py +++ b/hooks/neutron_hooks.py @@ -64,6 +64,7 @@ from neutron_utils import ( assess_status, install_systemd_override, configure_apparmor, + write_vendordata, ) hooks = Hooks() @@ -118,6 +119,9 @@ def config_changed(): if sysctl_dict: create_sysctl(sysctl_dict, '/etc/sysctl.d/50-quantum-gateway.conf') + if config('vendor-data'): + write_vendordata(config('vendor-data')) + # Re-run joined hooks as config might have changed for r_id in relation_ids('amqp'): amqp_joined(relation_id=r_id) diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index 0d71fd70..1417e8b7 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -1,4 +1,5 @@ import os +import json import shutil import subprocess from shutil import copy2 @@ -68,6 +69,7 @@ from neutron_contexts import ( CORE_PLUGIN, OVS, NSX, N1KV, OVS_ODL, NeutronGatewayContext, L3AgentContext, + NovaMetadataContext, ) from charmhelpers.contrib.openstack.neutron import ( parse_bridge_mappings, @@ -79,6 +81,7 @@ from copy import deepcopy def valid_plugin(): return config('plugin') in CORE_PLUGIN + NEUTRON_COMMON = 'neutron-common' VERSION_PACKAGE = NEUTRON_COMMON @@ -262,6 +265,7 @@ def determine_l3ha_packages(): def use_l3ha(): return NeutronAPIContext()()['enable_l3ha'] + EXT_PORT_CONF = '/etc/init/ext-port.conf' PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf' STOPPED_SERVICES = ['os-charm-phy-nic-mtu', 'ext-port'] @@ -293,7 +297,8 @@ NOVA_CONFIG_FILES = { SyslogContext(), context.WorkerConfigContext(), context.ZeroMQContext(), - context.NotificationDriverContext()], + context.NotificationDriverContext(), + NovaMetadataContext()], 'services': ['nova-api-metadata'] }, NOVA_API_METADATA_AA_PROFILE_PATH: { @@ -968,3 +973,17 @@ def configure_apparmor(): profiles.append(NEUTRON_LBAASV2_AA_PROFILE) for profile in profiles: context.AppArmorContext(profile).setup_aa_profile() + + +VENDORDATA_FILE = '/etc/nova/vendor_data.json' + + +def write_vendordata(vdata): + try: + json_vdata = json.loads(vdata) + except (TypeError, json.decoder.JSONDecodeError) as e: + log('Error decoding vendor-data. {}'.format(e), level=ERROR) + return False + with open(VENDORDATA_FILE, 'w') as vdata_file: + vdata_file.write(json.dumps(json_vdata, sort_keys=True, indent=2)) + return True diff --git a/templates/mitaka/nova.conf b/templates/mitaka/nova.conf index d7ab35c4..1ee18601 100644 --- a/templates/mitaka/nova.conf +++ b/templates/mitaka/nova.conf @@ -18,6 +18,11 @@ network_api_class=nova.network.neutronv2.api.API use_neutron = True metadata_workers = {{ workers }} +{% if vendor_data -%} +vendordata_driver = nova.api.metadata.vendordata_json.JsonFileVendorData +vendordata_jsonfile_path = /etc/nova/vendor_data.json +{% endif -%} + [neutron] url={{ quantum_url }} auth_url={{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }} diff --git a/templates/newton/nova.conf b/templates/newton/nova.conf new file mode 100644 index 00000000..8c6eb8d0 --- /dev/null +++ b/templates/newton/nova.conf @@ -0,0 +1,48 @@ +# newton +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +logdir=/var/log/nova +state_path=/var/lib/nova +root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf +debug = {{ debug }} +verbose= {{ verbose }} +use_syslog = {{ use_syslog }} +api_paste_config=/etc/nova/api-paste.ini +enabled_apis=metadata +multi_host=True +# Access to neutron API services +network_api_class=nova.network.neutronv2.api.API +use_neutron = True +metadata_workers = {{ workers }} + +{% if vendor_data or vendor_data_url -%} +[api] +vendordata_providers = {{ vendordata_providers }} +{% if vendor_data -%} +vendordata_jsonfile_path = /etc/nova/vendor_data.json +{% endif -%} +{% if vendor_data_url -%} +vendordata_dynamic_targets = {{ vendor_data_url }} +{% endif -%} +{% endif -%} + +[neutron] +url={{ quantum_url }} +auth_url={{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }} +auth_type=password +project_domain_name=default +user_domain_name=default +region={{ region }} +project_name={{ service_tenant }} +username={{ service_username }} +password={{ service_password }} +service_metadata_proxy=True +metadata_proxy_shared_secret={{ shared_secret }} + +{% include "section-rabbitmq-oslo" %} + +[oslo_concurrency] +lock_path=/var/lock/nova diff --git a/unit_tests/test_neutron_contexts.py b/unit_tests/test_neutron_contexts.py index 6baeeb1d..ca86eb57 100644 --- a/unit_tests/test_neutron_contexts.py +++ b/unit_tests/test_neutron_contexts.py @@ -18,6 +18,7 @@ TO_PATCH = [ 'eligible_leader', 'unit_get', 'network_get_primary_address', + 'os_release', ] @@ -335,3 +336,54 @@ class TestMisc(CharmTestCase): self.config.return_value = 'ovs' self.assertEqual(neutron_contexts.core_plugin(), neutron_contexts.NEUTRON_ML2_PLUGIN) + + +class TestNovaMetadataContext(CharmTestCase): + + def setUp(self): + super(TestNovaMetadataContext, self).setUp(neutron_contexts, + TO_PATCH) + self.config.side_effect = self.test_config.get + + def test_vendordata_static(self): + _vdata = '{"good": "json"}' + self.os_release.return_value = 'pike' + + self.test_config.set('vendor-data', _vdata) + ctxt = neutron_contexts.NovaMetadataContext()() + + self.assertTrue(ctxt['vendor_data']) + self.assertEqual(ctxt['vendordata_providers'], ['StaticJSON']) + + def test_vendordata_dynamic(self): + _vdata_url = 'http://example.org/vdata' + self.os_release.return_value = 'pike' + + self.test_config.set('vendor-data-url', _vdata_url) + ctxt = neutron_contexts.NovaMetadataContext()() + + self.assertEqual(ctxt['vendor_data_url'], _vdata_url) + self.assertEqual(ctxt['vendordata_providers'], ['DynamicJSON']) + + def test_vendordata_static_and_dynamic(self): + _vdata = '{"good": "json"}' + _vdata_url = 'http://example.org/vdata' + self.os_release.return_value = 'pike' + + self.test_config.set('vendor-data', _vdata) + self.test_config.set('vendor-data-url', _vdata_url) + ctxt = neutron_contexts.NovaMetadataContext()() + + self.assertTrue(ctxt['vendor_data']) + self.assertEqual(ctxt['vendor_data_url'], _vdata_url) + self.assertEqual(ctxt['vendordata_providers'], ['StaticJSON', + 'DynamicJSON']) + + def test_vendordata_mitaka(self): + _vdata_url = 'http://example.org/vdata' + self.os_release.return_value = 'mitaka' + + self.test_config.set('vendor-data-url', _vdata_url) + ctxt = neutron_contexts.NovaMetadataContext()() + + self.assertEqual(ctxt, {'vendordata_providers': []}) diff --git a/unit_tests/test_neutron_utils.py b/unit_tests/test_neutron_utils.py index f54c7abf..e44606db 100644 --- a/unit_tests/test_neutron_utils.py +++ b/unit_tests/test_neutron_utils.py @@ -11,6 +11,10 @@ from test_utils import ( CharmTestCase ) +from test_neutron_contexts import ( + patch_open +) + TO_PATCH = [ 'config', 'get_os_codename_install_source', @@ -717,6 +721,21 @@ class TestNeutronUtils(CharmTestCase): for config in EXC_CONFIG: self.assertTrue(config not in actual_configs) + def test_write_valid_json_vendordata(self): + _jdata = '{"good": "json"}' + _tdata = '{\n "good": "json"\n}' + with patch_open() as (_open, _file): + self.assertEqual(neutron_utils.write_vendordata(_jdata), True) + _open.assert_called_with(neutron_utils.VENDORDATA_FILE, 'w') + _file.write.assert_called_with(_tdata) + + @patch('json.loads') + def test_write_invalid_json_vendordata(self, _json_loads): + _jdata = '{ bad json }' + _json_loads.side_effect = TypeError + with patch_open() as (_open, _file): + self.assertEqual(neutron_utils.write_vendordata(_jdata), False) + network_context = { 'service_username': 'foo',