From ea39e9e5ed1e5403df09b67460456efd20ca2447 Mon Sep 17 00:00:00 2001
From: Michal Rostecki <michal.rostecki@gmail.com>
Date: Tue, 22 Mar 2016 14:43:38 +0100
Subject: [PATCH] Parse and render yaml files in the proper order

This commit introduces a clear order of loading jvars:
- passwords.yml
- globals.yml
- custom variables based on globals.yml
- config/<service>/all.yml
- config/<service/[...]

It also introduces a mechanism of jinja2 templating yaml
files by themselves, which means that one variable in yaml
file can be reused in the next variables inside the same file.

TrivialFix

Change-Id: I1a7a61e90963d396702f5e0c14eb476d3ccd21bf
---
 config/all.yml                               | 10 +++
 etc/globals.yml                              |  8 ---
 kolla_mesos/common/jinja_utils.py            | 49 +++++++++++++++
 kolla_mesos/configuration.py                 |  4 +-
 kolla_mesos/service.py                       | 65 ++++++++++++++------
 kolla_mesos/tests/common/test_jinja_utils.py | 29 +++++++++
 kolla_mesos/tests/test_service.py            | 31 ++++++++++
 7 files changed, 167 insertions(+), 29 deletions(-)
 create mode 100644 kolla_mesos/tests/common/test_jinja_utils.py

diff --git a/config/all.yml b/config/all.yml
index d17781b8..bd1b7ac0 100644
--- a/config/all.yml
+++ b/config/all.yml
@@ -126,6 +126,16 @@ openstack_region_name: "RegionOne"
 # Valid options are [ novnc, spice ]
 nova_console: "novnc"
 
+
+####################
+# Constraints
+####################
+controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
+compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
+controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
+storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
+
+
 ####################
 # Mesos-dns hosts
 ####################
diff --git a/etc/globals.yml b/etc/globals.yml
index 00ad18f5..07dc1078 100644
--- a/etc/globals.yml
+++ b/etc/globals.yml
@@ -42,14 +42,6 @@ controller_nodes: "1"
 compute_nodes: "1"
 storage_nodes: "1"
 
-####################
-# Constraints
-####################
-controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
-compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
-controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
-storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
-
 ####################
 # OpenStack options
 ####################
diff --git a/kolla_mesos/common/jinja_utils.py b/kolla_mesos/common/jinja_utils.py
index 6f9010cd..b81c45bc 100644
--- a/kolla_mesos/common/jinja_utils.py
+++ b/kolla_mesos/common/jinja_utils.py
@@ -10,17 +10,39 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import logging
 import os
 
 import jinja2
 from jinja2 import meta
+import six
+import yaml
 
 from kolla_mesos.common import type_utils
 
 LOG = logging.getLogger(__name__)
 
 
+# Customize PyYAML library to return the OrderedDict. That is needed, because
+# when iterating on dict, we reuse its previous values when processing the
+# next values and the order has to be preserved.
+
+def ordered_dict_constructor(loader, node):
+    """OrderedDict constructor for PyYAML."""
+    return collections.OrderedDict(loader.construct_pairs(node))
+
+
+def ordered_dict_representer(dumper, data):
+    """Representer for PyYAML which is able to work with OrderedDict."""
+    return dumper.represent_dict(data.items())
+
+
+yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                     ordered_dict_constructor)
+yaml.add_representer(collections.OrderedDict, ordered_dict_representer)
+
+
 def jinja_render(fullpath, global_config, extra=None):
     variables = global_config
     if extra:
@@ -51,3 +73,30 @@ def jinja_find_required_variables(fullpath):
                                               os.path.basename(fullpath))[0]
     parsed_content = myenv.parse(template_source)
     return meta.find_undeclared_variables(parsed_content)
+
+
+def dict_jinja_render(raw_dict, jvars):
+    """Renders dict with jinja2 using provided variables and itself.
+
+    By using itself, we mean reusing the previous values from dict for the
+    potential render of the next value in dict.
+    """
+    for key, value in raw_dict.items():
+        if isinstance(value, six.string_types):
+            value = jinja_render_str(value, jvars)
+        elif isinstance(value, dict):
+            value = dict_jinja_render(value, jvars)
+        jvars[key] = value
+
+
+def yaml_jinja_render(filename, jvars):
+    """Parses YAML file and templates it with jinja2.
+
+    1. YAML file is rendered by jinja2 based on the provided variables.
+    2. Rendered file is parsed.
+    3. The every element dictionary being a result of parsing is rendered again
+       with itself.
+    """
+    with open(filename, 'r') as yaml_file:
+        raw_dict = yaml.load(yaml_file)
+    dict_jinja_render(raw_dict, jvars)
diff --git a/kolla_mesos/configuration.py b/kolla_mesos/configuration.py
index c30f1fe5..cda345d5 100644
--- a/kolla_mesos/configuration.py
+++ b/kolla_mesos/configuration.py
@@ -155,13 +155,13 @@ def apply_deployment_vars(jvars):
             'controller_compute_constraints':
             controller_compute_constraints,
             'storage_constraints': storage_constraints
-        })
+        }, force=True)
     jvars.update({
         'controller_nodes': str(controller_nodes),
         'compute_nodes': str(compute_nodes),
         'storage_nodes': str(storage_nodes),
         'all_nodes': str(all_nodes)
-    })
+    }, force=True)
 
 
 def get_marathon_framework(jvars):
diff --git a/kolla_mesos/service.py b/kolla_mesos/service.py
index 6183b75d..239927f5 100644
--- a/kolla_mesos/service.py
+++ b/kolla_mesos/service.py
@@ -465,37 +465,64 @@ def _load_variables_from_zk(zk):
     return variables
 
 
+class JvarsDict(dict):
+    """Dict which can contain the 'global_vars' which are always preserved.
+
+    They cannot be be overriden by any update nor single item setting.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(JvarsDict, self).__init__(*args, **kwargs)
+        self.global_vars = {}
+
+    def __setitem__(self, key, value, force=False):
+        if not force and key in self.global_vars:
+            return
+        return super(JvarsDict, self).__setitem__(key, value)
+
+    def set_force(self, key, value):
+        """Sets the variable even if it will override a global variable."""
+        return self.__setitem__(key, value, force=True)
+
+    def update(self, other_dict, force=False):
+        if not force:
+            other_dict = {key: value for key, value in other_dict.items()
+                          if key not in self.global_vars}
+        super(JvarsDict, self).update(other_dict)
+
+    def set_global_vars(self, global_vars):
+        self.update(global_vars)
+        self.global_vars = global_vars
+
+
 def _load_variables_from_file(service_dir, project_name):
     config_dir = os.path.join(service_dir, '..', 'config')
-    with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
-        global_vars = yaml.load(gf)
+    jvars = JvarsDict()
     with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
-        global_vars.update(yaml.load(gf))
+        jvars.set_global_vars(yaml.load(gf))
+    with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
+        jvars.update(yaml.load(gf))
+    # Apply the basic variables that aren't defined in any config file.
+    jvars.update({
+        'deployment_id': CONF.kolla.deployment_id,
+        'node_config_directory': '',
+        'timestamp': str(time.time())
+    })
+    # Get the exact marathon framework name.
+    config.get_marathon_framework(jvars)
     # all.yml file uses some its variables to template itself by jinja2,
     # so its raw content is used to template the file
     all_yml_name = os.path.join(config_dir, 'all.yml')
-    with open(all_yml_name) as af:
-        raw_vars = yaml.load(af)
-    raw_vars.update(global_vars)
-    jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
-    jvars.update(global_vars)
+    jinja_utils.yaml_jinja_render(all_yml_name, jvars)
+    # Apply the dynamic deployment variables.
+    config.apply_deployment_vars(jvars)
 
     proj_yml_name = os.path.join(config_dir, project_name,
                                  'defaults', 'main.yml')
     if os.path.exists(proj_yml_name):
-        proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
-                                                       jvars))
-        jvars.update(proj_vars)
+        jinja_utils.yaml_jinja_render(proj_yml_name, jvars)
     else:
         LOG.warning('Path missing %s' % proj_yml_name)
-    # Add deployment_id
-    jvars.update({'deployment_id': CONF.kolla.deployment_id})
-    # override node_config_directory to empty
-    jvars.update({'node_config_directory': ''})
-    # Add timestamp
-    jvars.update({'timestamp': str(time.time())})
-    config.apply_deployment_vars(jvars)
-    config.get_marathon_framework(jvars)
     return jvars
 
 
diff --git a/kolla_mesos/tests/common/test_jinja_utils.py b/kolla_mesos/tests/common/test_jinja_utils.py
new file mode 100644
index 00000000..10f36df6
--- /dev/null
+++ b/kolla_mesos/tests/common/test_jinja_utils.py
@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+from kolla_mesos.common import jinja_utils
+from kolla_mesos.tests import base
+
+
+class TestJinjaUtils(base.BaseTestCase):
+
+    def test_dict_jinja_render(self):
+        raw_dict = collections.OrderedDict([
+            ('first_key', '{{ test_var }}_test',),
+            ('second_key', '{{ first_key }}_test'),
+        ])
+        jvars = {'test_var': 'test'}
+        jinja_utils.dict_jinja_render(raw_dict, jvars)
+        self.assertEqual(jvars['first_key'], 'test_test')
+        self.assertEqual(jvars['second_key'], 'test_test_test')
diff --git a/kolla_mesos/tests/test_service.py b/kolla_mesos/tests/test_service.py
index 2eccb31c..ae332e39 100644
--- a/kolla_mesos/tests/test_service.py
+++ b/kolla_mesos/tests/test_service.py
@@ -32,6 +32,37 @@ class TestAPI(base.BaseTestCase):
         self.addCleanup(self.client.close)
         cfg.CONF.set_override('deployment_id', 'did', group='kolla')
 
+    def test_jvars_dict(self):
+        jvars = service.JvarsDict(non_global1='old_value',
+                                  non_global2='old_value')
+        jvars.set_global_vars({'global1': 'old_value',
+                               'global2': 'old_value'})
+
+        jvars.update({'global1': 'new_value',
+                      'global2': 'new_value',
+                      'non_global1': 'new_value',
+                      'non_global2': 'new_value'})
+        self.assertDictEqual({'global1': 'old_value',
+                              'global2': 'old_value',
+                              'non_global1': 'new_value',
+                              'non_global2': 'new_value'}, jvars)
+
+        jvars['global1'] = 'newer_value'
+        jvars['global2'] = 'newer_value'
+        jvars['non_global1'] = 'newer_value'
+        jvars['non_global2'] = 'newer_value'
+        self.assertDictEqual({'global1': 'old_value',
+                              'global2': 'old_value',
+                              'non_global1': 'newer_value',
+                              'non_global2': 'newer_value'}, jvars)
+
+        jvars.set_force('global1', 'force_override')
+        jvars.update({'global2': 'force_override'}, force=True)
+        self.assertDictEqual({'global1': 'force_override',
+                              'global2': 'force_override',
+                              'non_global1': 'newer_value',
+                              'non_global2': 'newer_value'}, jvars)
+
     @mock.patch.object(service.config, 'get_marathon_framework')
     @mock.patch.object(service.config, 'apply_deployment_vars')
     @mock.patch.object(service.MarathonApp, 'run')