Modify j2 templating to allow role files generation

Currently we only allow transformation of a single *.j2.yaml
file into a *.yaml file - this adds the option to specify a
*.role.j2.yaml file, which will be rendered once per role to create
multiple files (one per role).

Co-Authored-By: Carlos Camacho <ccamacho@redhat.com>

Change-Id: I9f920e191344040a564214f3f9a1147b265e9ff3
Partial-Bug: #1626976

tripleo_common/actions/templates.py

@@ -68,6 +68,30 @@ class ProcessTemplatesAction(base.TripleOAction):
         super(ProcessTemplatesAction, self).__init__()
         self.container = container
 
+    def _j2_render_and_put(self, j2_template, j2_data, outfile_name=None):
+        swift = self._get_object_client()
+        yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
+
+        try:
+            # Render the j2 template
+            template = jinja2.Environment().from_string(j2_template)
+            r_template = template.render(**j2_data)
+        except jinja2.exceptions.TemplateError as ex:
+            error_msg = ("Error rendering template %s : %s"
+                         % (yaml_f, six.text_type(ex)))
+            LOG.error(error_msg)
+            raise Exception(error_msg)
+        try:
+            # write the template back to the plan container
+            LOG.info("Writing rendered template %s" % yaml_f)
+            swift.put_object(
+                self.container, yaml_f, r_template)
+        except swiftexceptions.ClientException as ex:
+            error_msg = ("Error storing file %s in container %s"
+                         % (yaml_f, self.container))
+            LOG.error(error_msg)
+            raise Exception(error_msg)
+
     def _process_custom_roles(self):
         swift = self._get_object_client()
         try:
@@ -78,6 +102,7 @@ class ProcessTemplatesAction(base.TripleOAction):
             LOG.info("No %s file found, skipping jinja templating"
                      % constants.OVERCLOUD_J2_ROLES_NAME)
             return
+
         try:
             # Iterate over all files in the plan container
             # we j2 render any with the .j2.yaml suffix
@@ -88,32 +113,32 @@ class ProcessTemplatesAction(base.TripleOAction):
             LOG.error(error_msg)
             raise Exception(error_msg)
 
+        role_names = [r.get('name') for r in role_data]
         for f in [f.get('name') for f in container_files[1]]:
-            # check to see if the file is a *.j2.yaml
-            # if it is, get it and template for roles
-            if f.endswith('.j2.yaml'):
+            # We do two templating passes here:
+            # 1. *.role.j2.yaml - we template just the role name
+            #    and create multiple files (one per role)
+            # 2. *.j2.yaml - we template with all roles_data,
+            #    and create one file common to all roles
+            if f.endswith('.role.j2.yaml'):
+                LOG.info("jinja2 rendering role template %s" % f)
+                j2_template = swift.get_object(self.container, f)[1]
+                LOG.info("jinja2 rendering roles %s" % ",".join(role_names))
+                for role in role_names:
+                    j2_data = {'role': role}
+                    LOG.info("jinja2 rendering role %s" % role)
+                    out_f = "-".join(
+                        [role.lower(),
+                         os.path.basename(f).replace('.role.j2.yaml',
+                                                     '.yaml')])
+                    out_f_path = os.path.join(os.path.dirname(f), out_f)
+                    self._j2_render_and_put(j2_template, j2_data, out_f_path)
+            elif f.endswith('.j2.yaml'):
                 LOG.info("jinja2 rendering %s" % f)
                 j2_template = swift.get_object(self.container, f)[1]
-
-                try:
-                    # Render the j2 template
-                    template = jinja2.Environment().from_string(j2_template)
-                    r_template = template.render(roles=role_data)
-                except jinja2.exceptions.TemplateError as ex:
-                    error_msg = ("Error rendering template %s : %s"
-                                 % (f, six.text_type(ex)))
-                    LOG.error(error_msg)
-                    raise Exception(error_msg)
-                try:
-                    # write the template back to the plan container
-                    yaml_f = f.replace('.j2.yaml', '.yaml')
-                    swift.put_object(
-                        self.container, yaml_f, r_template)
-                except swiftexceptions.ClientException as ex:
-                    error_msg = ("Error storing file %s in container %s"
-                                 % (yaml_f, self.container))
-                    LOG.error(error_msg)
-                    raise Exception(error_msg)
+                j2_data = {'roles': role_data}
+                out_f = f.replace('.j2.yaml', '.yaml')
+                self._j2_render_and_put(j2_template, j2_data, out_f)
 
     def run(self):
         error_text = None

tripleo_common/tests/actions/test_templates.py

@@ -20,7 +20,6 @@ from tripleo_common.actions import templates
 from tripleo_common import constants
 from tripleo_common.tests import base
 
-
 JINJA_SNIPPET = """
 # Jinja loop for Role in role_data.yaml
 {% for role in roles %}
@@ -40,7 +39,6 @@ ROLE_DATA_YAML = """
   name: Controller
 """
 
-
 EXPECTED_JINJA_RESULT = """
 # Jinja loop for Role in role_data.yaml
 
@@ -55,6 +53,12 @@ EXPECTED_JINJA_RESULT = """
       DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
 """
 
+JINJA_SNIPPET_CONFIG = """
+outputs:
+  OS::stack_id:
+    description: The software config which runs puppet on the {{role}} role
+    value: {get_resource: {{role}}PuppetConfigImpl}"""
+
 
 class UploadTemplatesActionTest(base.TestCase):
 
@@ -171,3 +175,23 @@ class ProcessTemplatesActionTest(base.TestCase):
         ]
         swift.put_object.assert_has_calls(
             put_object_mock_calls, any_order=True)
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    @mock.patch('mistral.context.ctx')
+    def test_j2_render_and_put(self, ctx_mock, get_obj_client_mock):
+
+        # setup swift
+        swift = mock.MagicMock()
+        swift.get_object = mock.MagicMock()
+        swift.get_container = mock.MagicMock()
+        get_obj_client_mock.return_value = swift
+
+        # Test
+        action = templates.ProcessTemplatesAction()
+        action._j2_render_and_put(JINJA_SNIPPET_CONFIG,
+                                  {'role': 'Controller'},
+                                  'controller-config.yaml')
+
+        action_result = swift.put_object._mock_mock_calls[0]
+
+        self.assertTrue("Controller" in str(action_result))