diff --git a/config/all.yml b/config/all.yml index d17781b8..bd1b7ac0 100644 --- a/config/all.yml +++ b/config/all.yml @@ -126,6 +126,16 @@ openstack_region_name: "RegionOne" # Valid options are [ novnc, spice ] nova_console: "novnc" + +#################### +# Constraints +#################### +controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]' +compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]' +controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]' +storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]' + + #################### # Mesos-dns hosts #################### diff --git a/etc/globals.yml b/etc/globals.yml index 00ad18f5..07dc1078 100644 --- a/etc/globals.yml +++ b/etc/globals.yml @@ -42,14 +42,6 @@ controller_nodes: "1" compute_nodes: "1" storage_nodes: "1" -#################### -# Constraints -#################### -controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]' -compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]' -controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]' -storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]' - #################### # OpenStack options #################### diff --git a/kolla_mesos/common/jinja_utils.py b/kolla_mesos/common/jinja_utils.py index 6f9010cd..b81c45bc 100644 --- a/kolla_mesos/common/jinja_utils.py +++ b/kolla_mesos/common/jinja_utils.py @@ -10,17 +10,39 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import logging import os import jinja2 from jinja2 import meta +import six +import yaml from kolla_mesos.common import type_utils LOG = logging.getLogger(__name__) +# Customize PyYAML library to return the OrderedDict. That is needed, because +# when iterating on dict, we reuse its previous values when processing the +# next values and the order has to be preserved. + +def ordered_dict_constructor(loader, node): + """OrderedDict constructor for PyYAML.""" + return collections.OrderedDict(loader.construct_pairs(node)) + + +def ordered_dict_representer(dumper, data): + """Representer for PyYAML which is able to work with OrderedDict.""" + return dumper.represent_dict(data.items()) + + +yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + ordered_dict_constructor) +yaml.add_representer(collections.OrderedDict, ordered_dict_representer) + + def jinja_render(fullpath, global_config, extra=None): variables = global_config if extra: @@ -51,3 +73,30 @@ def jinja_find_required_variables(fullpath): os.path.basename(fullpath))[0] parsed_content = myenv.parse(template_source) return meta.find_undeclared_variables(parsed_content) + + +def dict_jinja_render(raw_dict, jvars): + """Renders dict with jinja2 using provided variables and itself. + + By using itself, we mean reusing the previous values from dict for the + potential render of the next value in dict. + """ + for key, value in raw_dict.items(): + if isinstance(value, six.string_types): + value = jinja_render_str(value, jvars) + elif isinstance(value, dict): + value = dict_jinja_render(value, jvars) + jvars[key] = value + + +def yaml_jinja_render(filename, jvars): + """Parses YAML file and templates it with jinja2. + + 1. YAML file is rendered by jinja2 based on the provided variables. + 2. Rendered file is parsed. + 3. The every element dictionary being a result of parsing is rendered again + with itself. + """ + with open(filename, 'r') as yaml_file: + raw_dict = yaml.load(yaml_file) + dict_jinja_render(raw_dict, jvars) diff --git a/kolla_mesos/configuration.py b/kolla_mesos/configuration.py index c30f1fe5..cda345d5 100644 --- a/kolla_mesos/configuration.py +++ b/kolla_mesos/configuration.py @@ -155,13 +155,13 @@ def apply_deployment_vars(jvars): 'controller_compute_constraints': controller_compute_constraints, 'storage_constraints': storage_constraints - }) + }, force=True) jvars.update({ 'controller_nodes': str(controller_nodes), 'compute_nodes': str(compute_nodes), 'storage_nodes': str(storage_nodes), 'all_nodes': str(all_nodes) - }) + }, force=True) def get_marathon_framework(jvars): diff --git a/kolla_mesos/service.py b/kolla_mesos/service.py index 6183b75d..239927f5 100644 --- a/kolla_mesos/service.py +++ b/kolla_mesos/service.py @@ -465,37 +465,64 @@ def _load_variables_from_zk(zk): return variables +class JvarsDict(dict): + """Dict which can contain the 'global_vars' which are always preserved. + + They cannot be be overriden by any update nor single item setting. + """ + + def __init__(self, *args, **kwargs): + super(JvarsDict, self).__init__(*args, **kwargs) + self.global_vars = {} + + def __setitem__(self, key, value, force=False): + if not force and key in self.global_vars: + return + return super(JvarsDict, self).__setitem__(key, value) + + def set_force(self, key, value): + """Sets the variable even if it will override a global variable.""" + return self.__setitem__(key, value, force=True) + + def update(self, other_dict, force=False): + if not force: + other_dict = {key: value for key, value in other_dict.items() + if key not in self.global_vars} + super(JvarsDict, self).update(other_dict) + + def set_global_vars(self, global_vars): + self.update(global_vars) + self.global_vars = global_vars + + def _load_variables_from_file(service_dir, project_name): config_dir = os.path.join(service_dir, '..', 'config') - with open(file_utils.find_config_file('passwords.yml'), 'r') as gf: - global_vars = yaml.load(gf) + jvars = JvarsDict() with open(file_utils.find_config_file('globals.yml'), 'r') as gf: - global_vars.update(yaml.load(gf)) + jvars.set_global_vars(yaml.load(gf)) + with open(file_utils.find_config_file('passwords.yml'), 'r') as gf: + jvars.update(yaml.load(gf)) + # Apply the basic variables that aren't defined in any config file. + jvars.update({ + 'deployment_id': CONF.kolla.deployment_id, + 'node_config_directory': '', + 'timestamp': str(time.time()) + }) + # Get the exact marathon framework name. + config.get_marathon_framework(jvars) # all.yml file uses some its variables to template itself by jinja2, # so its raw content is used to template the file all_yml_name = os.path.join(config_dir, 'all.yml') - with open(all_yml_name) as af: - raw_vars = yaml.load(af) - raw_vars.update(global_vars) - jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars)) - jvars.update(global_vars) + jinja_utils.yaml_jinja_render(all_yml_name, jvars) + # Apply the dynamic deployment variables. + config.apply_deployment_vars(jvars) proj_yml_name = os.path.join(config_dir, project_name, 'defaults', 'main.yml') if os.path.exists(proj_yml_name): - proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name, - jvars)) - jvars.update(proj_vars) + jinja_utils.yaml_jinja_render(proj_yml_name, jvars) else: LOG.warning('Path missing %s' % proj_yml_name) - # Add deployment_id - jvars.update({'deployment_id': CONF.kolla.deployment_id}) - # override node_config_directory to empty - jvars.update({'node_config_directory': ''}) - # Add timestamp - jvars.update({'timestamp': str(time.time())}) - config.apply_deployment_vars(jvars) - config.get_marathon_framework(jvars) return jvars diff --git a/kolla_mesos/tests/common/test_jinja_utils.py b/kolla_mesos/tests/common/test_jinja_utils.py new file mode 100644 index 00000000..10f36df6 --- /dev/null +++ b/kolla_mesos/tests/common/test_jinja_utils.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from kolla_mesos.common import jinja_utils +from kolla_mesos.tests import base + + +class TestJinjaUtils(base.BaseTestCase): + + def test_dict_jinja_render(self): + raw_dict = collections.OrderedDict([ + ('first_key', '{{ test_var }}_test',), + ('second_key', '{{ first_key }}_test'), + ]) + jvars = {'test_var': 'test'} + jinja_utils.dict_jinja_render(raw_dict, jvars) + self.assertEqual(jvars['first_key'], 'test_test') + self.assertEqual(jvars['second_key'], 'test_test_test') diff --git a/kolla_mesos/tests/test_service.py b/kolla_mesos/tests/test_service.py index 2eccb31c..ae332e39 100644 --- a/kolla_mesos/tests/test_service.py +++ b/kolla_mesos/tests/test_service.py @@ -32,6 +32,37 @@ class TestAPI(base.BaseTestCase): self.addCleanup(self.client.close) cfg.CONF.set_override('deployment_id', 'did', group='kolla') + def test_jvars_dict(self): + jvars = service.JvarsDict(non_global1='old_value', + non_global2='old_value') + jvars.set_global_vars({'global1': 'old_value', + 'global2': 'old_value'}) + + jvars.update({'global1': 'new_value', + 'global2': 'new_value', + 'non_global1': 'new_value', + 'non_global2': 'new_value'}) + self.assertDictEqual({'global1': 'old_value', + 'global2': 'old_value', + 'non_global1': 'new_value', + 'non_global2': 'new_value'}, jvars) + + jvars['global1'] = 'newer_value' + jvars['global2'] = 'newer_value' + jvars['non_global1'] = 'newer_value' + jvars['non_global2'] = 'newer_value' + self.assertDictEqual({'global1': 'old_value', + 'global2': 'old_value', + 'non_global1': 'newer_value', + 'non_global2': 'newer_value'}, jvars) + + jvars.set_force('global1', 'force_override') + jvars.update({'global2': 'force_override'}, force=True) + self.assertDictEqual({'global1': 'force_override', + 'global2': 'force_override', + 'non_global1': 'newer_value', + 'non_global2': 'newer_value'}, jvars) + @mock.patch.object(service.config, 'get_marathon_framework') @mock.patch.object(service.config, 'apply_deployment_vars') @mock.patch.object(service.MarathonApp, 'run')