From 2c0091d23d57ba2ffd8b65f944cda44f3548eac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Douglas=20Mendiz=C3=A1bal?= Date: Tue, 11 Dec 2018 15:49:49 -0600 Subject: [PATCH] Handle compact services on multiple lines This patch adds logic to handle compact service metadata that has been split into multiple lines to avoid hitting the metadata size limit. Co-Authored-By: Grzegorz Grasza Change-Id: Ida39f5768c67f982b2fe316f6fae4988a74c8534 --- novajoin/join.py | 29 +++++++++++---------- novajoin/notifications.py | 32 ++++++++++++------------ novajoin/tests/unit/test_util.py | 43 ++++++++++++++++++++++++++++++++ novajoin/util.py | 34 ++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 32 deletions(-) create mode 100644 novajoin/tests/unit/test_util.py diff --git a/novajoin/join.py b/novajoin/join.py index 9345663..5af53c1 100644 --- a/novajoin/join.py +++ b/novajoin/join.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. -import json import logging import traceback import uuid @@ -26,7 +25,7 @@ from novajoin.glance import get_default_image_service from novajoin.ipa import IPAClient from novajoin import keystone_client from novajoin.nova import get_instance -from novajoin.util import get_fqdn +from novajoin import util CONF = cfg.CONF @@ -200,7 +199,7 @@ class JoinController(Controller): ipaotp = uuid.uuid4().hex - data['hostname'] = get_fqdn(hostname_short, project_name) + data['hostname'] = util.get_fqdn(hostname_short, project_name) _, realm = self.ipaclient.get_host_and_realm() data['krb_realm'] = realm @@ -220,10 +219,11 @@ class JoinController(Controller): if key.startswith('managed_service_')] if managed_services: self.handle_services(data['hostname'], managed_services) - # compact json format - if 'compact_services' in metadata: - self.handle_compact_services(hostname_short, - metadata.get('compact_services')) + + compact_services = util.get_compact_services(metadata) + if compact_services: + self.handle_compact_services(hostname_short, compact_services) + self.ipaclient.flush_batch_operation() return data @@ -250,13 +250,13 @@ class JoinController(Controller): self.ipaclient.service_add_host(principal, base_host) - def handle_compact_services(self, base_host_short, service_repr_json): + def handle_compact_services(self, base_host_short, service_repr): """Make any host/principal assignments passed from metadata - This takes a representation of the services and networks where the - services are listening on, and forms appropriate hostnames/service - principals based on this information. The representation looks as the - following: + This takes a dictionary representation of the services and networks + where the services are listening on, and forms appropriate + hostnames/service principals based on this information. + The dictionary representation looks as the following: { "service1": [ @@ -286,15 +286,14 @@ class JoinController(Controller): """ LOG.debug("In handle compact services") - service_repr = json.loads(service_repr_json) hosts_found = list() services_found = list() - base_host = get_fqdn(base_host_short) + base_host = util.get_fqdn(base_host_short) for service_name, net_list in service_repr.items(): for network in net_list: host_short = "%s.%s" % (base_host_short, network) - principal_host = get_fqdn(host_short) + principal_host = util.get_fqdn(host_short) principal = "%s/%s" % (service_name, principal_host) # add host if not present diff --git a/novajoin/notifications.py b/novajoin/notifications.py index bfd6829..d821e2e 100644 --- a/novajoin/notifications.py +++ b/novajoin/notifications.py @@ -17,13 +17,16 @@ # notification_topic = notifications # notify_on_state_change = vm_state -import json import sys import time import glanceclient as glance_client from neutronclient.v2_0 import client as neutron_client from novaclient import client as nova_client +from oslo_log import log as logging +import oslo_messaging +from oslo_serialization import jsonutils + from novajoin import config from novajoin import exception from novajoin.ipa import IPAClient @@ -31,11 +34,7 @@ from novajoin import join from novajoin.keystone_client import get_session from novajoin.keystone_client import register_keystoneauth_opts from novajoin.nova import get_instance -from novajoin.util import get_domain -from novajoin.util import get_fqdn -from oslo_log import log as logging -import oslo_messaging -from oslo_serialization import jsonutils +from novajoin import util CONF = config.CONF @@ -169,10 +168,12 @@ class NotificationEndpoint(object): if key.startswith('managed_service_')] if managed_services: join_controller.handle_services(hostname, managed_services) - # compact json format - if 'compact_services' in payload_metadata: + + compact_services = util.get_compact_services(payload_metadata) + if compact_services: join_controller.handle_compact_services( - hostname_short, payload_metadata.get('compact_services')) + hostname_short, compact_services) + ipa.flush_batch_operation() @event_handlers('compute.instance.delete.end') @@ -262,16 +263,16 @@ class NotificationEndpoint(object): if metadata is None: return - if 'compact_services' in metadata: + compact_services = util.get_compact_services(metadata) + if compact_services: self.handle_compact_services(ipa, hostname_short, - metadata.get('compact_services')) + compact_services) managed_services = [metadata[key] for key in metadata.keys() if key.startswith('managed_service_')] if managed_services: self.handle_managed_services(ipa, managed_services) - def handle_compact_services(self, ipa, host_short, - service_repr_json): + def handle_compact_services(self, ipa, host_short, service_repr): """Reconstructs and removes subhosts for compact services. Data looks like this: @@ -286,14 +287,13 @@ class NotificationEndpoint(object): integrity. """ LOG.debug("In handle compact services") - service_repr = json.loads(service_repr_json) hosts_found = list() ipa.start_batch_operation() for service_name, net_list in service_repr.items(): for network in net_list: host = "%s.%s" % (host_short, network) - principal_host = get_fqdn(host) + principal_host = util.get_fqdn(host) # remove host if principal_host not in hosts_found: @@ -330,7 +330,7 @@ class NotificationEndpoint(object): def _generate_hostname(self, hostname): # FIXME: Don't re-calculate the hostname, fetch it from somewhere project = 'foo' - domain = get_domain() + domain = util.get_domain() if CONF.project_subdomain: host = '%s.%s.%s' % (hostname, project, domain) else: diff --git a/novajoin/tests/unit/test_util.py b/novajoin/tests/unit/test_util.py new file mode 100644 index 0000000..4f155e1 --- /dev/null +++ b/novajoin/tests/unit/test_util.py @@ -0,0 +1,43 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit Tests for util functions +""" + +import json +import testtools + +from novajoin import util + + +class TestUtil(testtools.TestCase): + + def setUp(self): + super(TestUtil, self).setUp() + + def test_get_compact_services(self): + result = {"http": ["internalapi", "ctlplane", "storage"], + "rabbitmq": ["internalapi", "ctlplane"]} + old_metadata = {"compact_services": json.dumps(result)} + new_metadata = { + "compact_service_http": json.dumps(result['http']), + "compact_service_rabbitmq": json.dumps(result['rabbitmq'])} + + self.assertDictEqual(util.get_compact_services(old_metadata), result) + + self.assertDictEqual(util.get_compact_services(new_metadata), result) + + def test_get_compact_services_empty(self): + self.assertIsNone(util.get_compact_services({})) diff --git a/novajoin/util.py b/novajoin/util.py index 1d4fc90..12159f8 100644 --- a/novajoin/util.py +++ b/novajoin/util.py @@ -14,10 +14,13 @@ """Utility functions shared between notify and server""" -from novajoin.errors import ConfigurationError +import json + from oslo_config import cfg from oslo_log import log as logging +import six +from novajoin.errors import ConfigurationError from novajoin.ipa import ipalib_imported if ipalib_imported: from ipalib import api @@ -54,3 +57,32 @@ def get_fqdn(hostname, project_name=None): return '%s.%s.%s' % (hostname, project_name, domain) else: return '%s.%s' % (hostname, domain) + + +def get_compact_services(metadata): + """Retrieve and convert the compact_services from instance metadata. + + This converts the new compact services format to the old/internal one. + The old format looks like: + + "compact_services": { + "http": ["internalapi", "ctlplane", "storage"], + "rabbitmq": ["internalapi", "ctlplane"] + } + + The new format contains service names inside the primary key: + + "compact_services_http": ["internalapi", "ctlplane", "storage"], + "compact_services_rabbitmq": ["internalapi", "ctlplane"] + """ + # compact key-per-service + compact_services = {key.split('_', 2)[-1]: json.loads(value) + for key, value in six.iteritems(metadata) + if key.startswith('compact_service_')} + if compact_services: + return compact_services + # legacy compact json format + if 'compact_services' in metadata: + return json.loads(metadata['compact_services']) + + return None