From 218582c4cc47d35181ce34a2b27f899fc1316be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Jens=C3=A5s?= Date: Sat, 23 Mar 2019 14:50:27 +0100 Subject: [PATCH] Make krb-service-principal metadata per-Role Not all roles are connected to all networks, there is no need to create metadata for networks not associated with the role. In edge/spine-and-leaf deployments the total number of composable networks used can be high. Passing all the networks we quickly go beyond the nova metadata fields size limit (each field cannot exceed 256 bytes). Also update tools/check-up-to-date.sh script to use the simple yaml-diff.py instead of diff. The env generator code will sort data, while jinja rendered environments are not sorted, thus need to diff the data in yaml not the text. Conflicts: environments/ssl/enable-internal-tls.j2.yaml Closes-Bug: #1821377 Change-Id: I5ae3bc845b0a6ad6986d44b14ff4b0737a9b033b (cherry picked from commit d5ecc1f6518d14f1ae6fefebd73533f11762bedf) --- .../role.role.j2.yaml} | 4 +-- ...l-tls.yaml => enable-internal-tls.j2.yaml} | 4 ++- .../role.role.j2.yaml} | 7 ++-- overcloud-resource-registry-puppet.j2.yaml | 8 +++-- overcloud.j2.yaml | 2 +- sample-env-generator/ssl.yaml | 6 +++- tools/check-up-to-date.sh | 2 +- tools/process-templates.py | 7 ++++ tools/yaml-diff.py | 32 +++++++++++++++++++ 9 files changed, 60 insertions(+), 12 deletions(-) rename common/{services.yaml => services/role.role.j2.yaml} (99%) rename environments/ssl/{enable-internal-tls.yaml => enable-internal-tls.j2.yaml} (90%) rename extraconfig/nova_metadata/{krb-service-principals.j2.yaml => krb-service-principals/role.role.j2.yaml} (93%) create mode 100755 tools/yaml-diff.py diff --git a/common/services.yaml b/common/services/role.role.j2.yaml similarity index 99% rename from common/services.yaml rename to common/services/role.role.j2.yaml index c904ad4042..d7b42fb898 100644 --- a/common/services.yaml +++ b/common/services/role.role.j2.yaml @@ -58,7 +58,7 @@ resources: type: OS::TripleO::LoggingConfiguration ServiceServerMetadataHook: - type: OS::TripleO::ServiceServerMetadataHook + type: OS::TripleO::{{role.name}}ServiceServerMetadataHook properties: RoleData: {get_attr: [ServiceChain, role_data]} @@ -70,7 +70,7 @@ resources: list_join: - "\n" - - str_replace: - template: {get_file: ../puppet/manifests/overcloud_common.pp} + template: {get_file: ../../puppet/manifests/overcloud_common.pp} params: __ROLE__: {get_param: RoleName} - yaql: diff --git a/environments/ssl/enable-internal-tls.yaml b/environments/ssl/enable-internal-tls.j2.yaml similarity index 90% rename from environments/ssl/enable-internal-tls.yaml rename to environments/ssl/enable-internal-tls.j2.yaml index d6b4aa8ad4..4b1018d4c1 100644 --- a/environments/ssl/enable-internal-tls.yaml +++ b/environments/ssl/enable-internal-tls.j2.yaml @@ -34,8 +34,10 @@ parameter_defaults: # End static parameters # ********************* resource_registry: - OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml OS::TripleO::Services::CertmongerUser: ../../puppet/services/certmonger-user.yaml OS::TripleO::Services::HAProxyInternalTLS: ../../puppet/services/haproxy-internal-tls-certmonger.yaml OS::TripleO::Services::IpaClient: ../../extraconfig/services/ipaclient.yaml OS::TripleO::Services::TLSProxyBase: ../../puppet/services/apache.yaml +{%- for role in roles %} + OS::TripleO::{{role.name}}ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/{{role.name.lower()}}-role.yaml +{%- endfor %} diff --git a/extraconfig/nova_metadata/krb-service-principals.j2.yaml b/extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml similarity index 93% rename from extraconfig/nova_metadata/krb-service-principals.j2.yaml rename to extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml index 633a5c0838..8f0edac25f 100644 --- a/extraconfig/nova_metadata/krb-service-principals.j2.yaml +++ b/extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml @@ -1,11 +1,11 @@ heat_template_version: rocky -description: 'Generates the relevant service principals for a server' +description: 'Generates the relevant service principals for a {{role.name}} server' parameters: RoleData: type: json description: the list containing the 'role_data' output for the ServiceChain -{%- for network in networks if network.vip|default(false) %} +{%- for network in networks if network.vip|default(false) and network.name in role.networks %} {%- if network.name == 'External' %} # Special case the External hostname param, which is CloudName CloudName: @@ -69,7 +69,7 @@ resources: data: metadata: {get_attr: [IncomingMetadataSettings, value]} fqdns: -{%- for network in networks if network.vip|default(false) %} +{%- for network in networks if network.vip|default(false) and network.name in role.networks %} {%- if network.name == 'External' %} external: {get_param: CloudName} {%- elif network.name == 'InternalApi' %} @@ -97,4 +97,3 @@ outputs: map_merge: - {get_attr: [IndividualServices, value]} - {get_attr: [CompactServices, value]} - diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 3ca43f8316..c0f833c702 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -33,7 +33,9 @@ resource_registry: # in the jinja loop OS::TripleO::Controller::Net::SoftwareConfig: net-config-bridge.yaml - OS::TripleO::ServiceServerMetadataHook: OS::Heat::None +{% for role in roles %} + OS::TripleO::{{role.name}}ServiceServerMetadataHook: OS::Heat::None +{%- endfor %} OS::TripleO::Server: OS::Nova::Server {% for role in roles %} @@ -102,7 +104,9 @@ resource_registry: OS::TripleO::WorkflowSteps: OS::Mistral::ExternalResource # services - OS::TripleO::Services: common/services.yaml +{%- for role in roles %} + OS::TripleO::{{role.name}}Services: common/services/{{role.name.lower()}}-role.yaml +{%- endfor %} OS::TripleO::Services::Aide: OS::Heat::None OS::TripleO::Services::Apache: puppet/services/apache.yaml OS::TripleO::Services::CACerts: puppet/services/ca-certs.yaml diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index 56d8691195..1db778cf42 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -410,7 +410,7 @@ resources: {% for role in roles %} # Resources generated for {{role.name}} Role {{role.name}}ServiceChain: - type: OS::TripleO::Services + type: OS::TripleO::{{role.name}}Services properties: Services: get_param: {{role.name}}Services diff --git a/sample-env-generator/ssl.yaml b/sample-env-generator/ssl.yaml index da9bac13bd..945f67b421 100644 --- a/sample-env-generator/ssl.yaml +++ b/sample-env-generator/ssl.yaml @@ -65,7 +65,11 @@ environments: OS::TripleO::Services::TLSProxyBase: ../../puppet/services/apache.yaml # Creates nova metadata that will create the extra service principals per # node. - OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml + OS::TripleO::ControllerServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/controller-role.yaml + OS::TripleO::ComputeServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/compute-role.yaml + OS::TripleO::BlockStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/blockstorage-role.yaml + OS::TripleO::ObjectStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/objectstorage-role.yaml + OS::TripleO::CephStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/cephstorage-role.yaml - name: ssl/inject-trust-anchor title: Inject SSL Trust Anchor on Overcloud Nodes description: | diff --git a/tools/check-up-to-date.sh b/tools/check-up-to-date.sh index eaa29d31c1..41a9c9f24f 100755 --- a/tools/check-up-to-date.sh +++ b/tools/check-up-to-date.sh @@ -18,7 +18,7 @@ cd $tmpdir file_list=$(find environments -type f) for f in $file_list; do - if ! diff -q $f $base/$f; then + if ! $base/tools/yaml-diff.py $f $base/$f; then echo "ERROR: $base/$f is not up to date" diff $f $base/$f retval=1 diff --git a/tools/process-templates.py b/tools/process-templates.py index 3ef6389c35..7abe231b93 100755 --- a/tools/process-templates.py +++ b/tools/process-templates.py @@ -320,9 +320,16 @@ def clean_templates(base_path, role_data_path, network_data_path): host_config_and_reboot_path = os.path.join( 'extraconfig', 'pre_network', '%s-host_config_and_reboot.yaml' % role['name'].lower()) + krb_service_principals_path = os.path.join( + 'extraconfig', 'nova_metadata', 'krb-service-principals', + '%s-role.yaml' % role['name'].lower()) + common_services_path = os.path.join( + 'common', 'services', '%s-role.yaml' % role['name'].lower()) delete(role_path) delete(host_config_and_reboot_path) + delete(krb_service_principals_path) + delete(common_services_path) nic_config_dir = os.path.join(base_path, 'network', 'config') for sample_nic_config_dir in os.listdir(nic_config_dir): diff --git a/tools/yaml-diff.py b/tools/yaml-diff.py new file mode 100755 index 0000000000..6001daf0e1 --- /dev/null +++ b/tools/yaml-diff.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import yaml + +if len(sys.argv) != 3: + raise RuntimeError('Not enough arguemnts') + +FILE_A = sys.argv[1] +FILE_B = sys.argv[2] + +with open(FILE_A, 'r') as file_a: + a = yaml.safe_load(file_a) + +with open(FILE_B, 'r') as file_b: + b = yaml.safe_load(file_b) + +if a != b: + sys.exit("Files are different") + +sys.exit(0)