Merge "Add support for jinja as userdata template engine"

cloudbaseinit/plugins/common/userdata.py

@@ -26,6 +26,7 @@ from cloudbaseinit.plugins.common import execcmd
 from cloudbaseinit.plugins.common.userdataplugins import factory
 from cloudbaseinit.plugins.common import userdatautils
 from cloudbaseinit.utils import encoding
+from cloudbaseinit.utils.template_engine import factory as template_factory
 from cloudbaseinit.utils import x509constants
 
 
@@ -53,7 +54,7 @@ class UserDataPlugin(base.BasePlugin):
             self._write_userdata(user_data, user_data_path)
 
         if CONF.process_userdata:
-            return self._process_user_data(user_data)
+            return self._process_user_data(user_data, service)
         return base.PLUGIN_EXECUTION_DONE, False
 
     @staticmethod
@@ -100,7 +101,7 @@ class UserDataPlugin(base.BasePlugin):
                                                    "The user data content is "
                                                    "either invalid or empty.")
 
-    def _process_user_data(self, user_data):
+    def _process_user_data(self, user_data, service):
         plugin_status = base.PLUGIN_EXECUTION_DONE
         reboot = False
         headers = self._get_headers(user_data)
@@ -122,7 +123,7 @@ class UserDataPlugin(base.BasePlugin):
 
             return plugin_status, reboot
         else:
-            return self._process_non_multi_part(user_data)
+            return self._process_non_multi_part(user_data, service)
 
     def _process_part(self, part, user_data_plugins, user_handlers):
         ret_val = None
@@ -186,8 +187,16 @@ class UserDataPlugin(base.BasePlugin):
         LOG.debug("Calling part handler \"__end__\" event")
         handler_func(None, "__end__", None, None)
 
-    def _process_non_multi_part(self, user_data):
+    def _process_non_multi_part(self, user_data, service):
         ret_val = None
+
+        template_engine = template_factory.get_template_engine(user_data)
+        if template_engine:
+            user_data = template_engine.render(
+                service.get_instance_data(),
+                user_data
+            )
+
         if user_data.startswith(b'#cloud-config'):
             user_data_plugins = factory.load_plugins()
             cloud_config_plugin = user_data_plugins.get('text/cloud-config')

cloudbaseinit/tests/plugins/common/test_userdata.py

@@ -162,8 +162,10 @@ class UserDataPluginTest(unittest.TestCase):
         mock_part = mock.MagicMock()
         mock_parse_mime.return_value = [mock_part]
         mock_process_part.return_value = (base.PLUGIN_EXECUTION_DONE, reboot)
+        mock_service = mock.MagicMock()
 
-        response = self._userdata._process_user_data(user_data=user_data)
+        response = self._userdata._process_user_data(user_data=user_data,
+                                                     service=mock_service)
 
         if user_data.startswith(b'Content-Type: multipart'):
             mock_load_plugins.assert_called_once_with()
@@ -172,7 +174,8 @@ class UserDataPluginTest(unittest.TestCase):
                                                       mock_load_plugins(), {})
             self.assertEqual((base.PLUGIN_EXECUTION_DONE, reboot), response)
         else:
-            mock_process_non_multi_part.assert_called_once_with(user_data)
+            mock_process_non_multi_part.assert_called_once_with(user_data,
+                                                                mock_service)
             self.assertEqual(mock_process_non_multi_part.return_value,
                              response)
 
@@ -313,8 +316,9 @@ class UserDataPluginTest(unittest.TestCase):
                 '.execute_user_data_script')
     def test_process_non_multi_part(self, mock_execute_user_data_script):
         user_data = b'fake'
+        service = mock.MagicMock()
         status, reboot = self._userdata._process_non_multi_part(
-            user_data=user_data)
+            user_data=user_data, service=service)
         mock_execute_user_data_script.assert_called_once_with(user_data)
         self.assertEqual(status, 1)
         self.assertFalse(reboot)
@@ -329,10 +333,11 @@ class UserDataPluginTest(unittest.TestCase):
         b2NhbGhvc3QwHhcNMTUwNjE1MTAyODUxWhcNMjUwNjEyMTAyODUxWjAbMRkwFwYD
         -----END CERTIFICATE-----
         ''').encode()
+        service = mock.MagicMock()
         with testutils.LogSnatcher('cloudbaseinit.plugins.'
                                    'common.userdata') as snatcher:
             status, reboot = self._userdata._process_non_multi_part(
-                user_data=user_data)
+                user_data=user_data, service=service)
 
         expected_logging = ['Found X509 certificate in userdata']
         self.assertFalse(mock_execute_user_data_script.called)
@@ -340,25 +345,54 @@ class UserDataPluginTest(unittest.TestCase):
         self.assertEqual(1, status)
         self.assertFalse(reboot)
 
+    @mock.patch('cloudbaseinit.utils.template_engine.factory.'
+                'get_template_engine')
     @mock.patch('cloudbaseinit.plugins.common.userdataplugins.factory.'
                 'load_plugins')
-    def test_process_non_multi_part_cloud_config(self, mock_load_plugins):
-        user_data = b'#cloud-config'
+    def _test_process_non_multi_part_cloud_config(self, mock_load_plugins,
+                                                  mock_load_templates,
+                                                  user_data,
+                                                  expected_userdata,
+                                                  template_renderer=None):
+        mock_service = mock.MagicMock()
         mock_return_value = mock.sentinel.return_value
         mock_cloud_config_plugin = mock.Mock()
         mock_cloud_config_plugin.process.return_value = mock_return_value
         mock_load_plugins.return_value = {
             'text/cloud-config': mock_cloud_config_plugin}
+        mock_load_templates.return_value = template_renderer
         status, reboot = self._userdata._process_non_multi_part(
-            user_data=user_data)
+            user_data=user_data, service=mock_service)
+
+        if template_renderer:
+            mock_load_plugins.assert_called_once_with()
+
+            (mock_cloud_config_plugin
+             .process_non_multipart
+             .assert_called_once_with(expected_userdata))
 
-        mock_load_plugins.assert_called_once_with()
-        (mock_cloud_config_plugin
-         .process_non_multipart
-         .assert_called_once_with(user_data))
         self.assertEqual(status, 1)
         self.assertFalse(reboot)
 
+    def test_process_non_multi_part_cloud_config(self):
+        user_data = b'#cloud-config'
+        self._test_process_non_multi_part_cloud_config(
+            user_data=user_data, expected_userdata=user_data)
+
+    def test_process_non_multi_part_cloud_config_jinja(self):
+        user_data = b'## template:jinja\n#cloud-config'
+        expected_userdata = b'#cloud-config'
+        mock_template_renderer = mock.MagicMock()
+        mock_template_renderer.render.return_value = expected_userdata
+        self._test_process_non_multi_part_cloud_config(
+            user_data=user_data, expected_userdata=expected_userdata,
+            template_renderer=mock_template_renderer)
+
+    def test_process_non_multi_part_no_valid_template(self):
+        user_data = b'## template:none'
+        self._test_process_non_multi_part_cloud_config(
+            user_data=user_data, expected_userdata=user_data)
+
 
 class TestCloudConfig(unittest.TestCase):
     @classmethod

cloudbaseinit/tests/utils/template_engine/test_base_template.py

@@ -0,0 +1,30 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+import unittest
+
+from cloudbaseinit.utils.template_engine import base_template as bt
+
+
+@ddt.ddt
+class TestBaseTemplateEngine(unittest.TestCase):
+
+    @ddt.data((b'', b''),
+              (b'## template:jinja test', b''),
+              (b'## template:jinja \ntest', b'test'))
+    @ddt.unpack
+    def test_remove_template_definition(self, template, expected_output):
+        output = bt.BaseTemplateEngine.remove_template_definition(template)
+        self.assertEqual(expected_output, output)

cloudbaseinit/tests/utils/template_engine/test_factory.py

@@ -0,0 +1,58 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+import unittest
+
+from cloudbaseinit.utils.template_engine import factory
+
+
+class FakeLoaderError(Exception):
+    pass
+
+
+class TestTemplateFactory(unittest.TestCase):
+
+    def test_get_template_engine_empty(self):
+        fake_userdata = b''
+        result = factory.get_template_engine(fake_userdata)
+        self.assertEqual(result, None)
+
+    def test_get_template_engine_no_match(self):
+        fake_userdata = b'no match'
+        result = factory.get_template_engine(fake_userdata)
+        self.assertEqual(result, None)
+
+    def test_get_template_engine_not_supported(self):
+        fake_userdata = b'## template:fake'
+        result = factory.get_template_engine(fake_userdata)
+        self.assertEqual(result, None)
+
+    @mock.patch('cloudbaseinit.utils.classloader.ClassLoader')
+    def test_get_template_engine(self, mock_class_loader):
+        fake_userdata = b'## template:jinja'
+        mock_load_class = mock_class_loader.return_value.load_class
+        self.assertEqual(mock_load_class.return_value.return_value,
+                         factory.get_template_engine(fake_userdata))
+
+    @mock.patch('cloudbaseinit.utils.classloader.ClassLoader')
+    def test_get_template_engine_class_not_found(self, mock_class_loader):
+        fake_userdata = b'## template:jinja'
+        mock_class_loader.return_value.load_class.side_effect = (
+            FakeLoaderError)
+        self.assertRaises(FakeLoaderError,
+                          factory.get_template_engine, fake_userdata)

cloudbaseinit/tests/utils/template_engine/test_jinja2_template.py

@@ -0,0 +1,82 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import unittest
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
+from cloudbaseinit.utils.template_engine.jinja2_template import (
+    Jinja2TemplateEngine)
+
+
+class TestJinja2TemplateEngine(unittest.TestCase):
+
+    @mock.patch('cloudbaseinit.utils.template_engine.base_template'
+                '.BaseTemplateEngine.remove_template_definition')
+    def _test_jinja_render_template(self, mock_remove_header,
+                                    fake_instance_data, expected_result,
+                                    fake_template = b'{{v1.local_hostname}}'):
+
+        mock_remove_header.return_value = fake_template
+
+        output = Jinja2TemplateEngine().render(fake_instance_data,
+                                               fake_template)
+
+        self.assertEqual(expected_result, output)
+
+    def test_jinja_render_template(self):
+        fake_instance_data = {
+            'v1': {
+                'local_hostname': 'fake_hostname'
+            }
+        }
+        expected_result = b'fake_hostname'
+        self._test_jinja_render_template(
+            fake_instance_data=fake_instance_data,
+            expected_result=expected_result)
+
+    def test_jinja_render_template_missing_variable(self):
+        fake_instance_data = {
+            'v1': {
+                'localhostname': 'fake_hostname'
+            }
+        }
+        expected_result = b'CI_MISSING_JINJA_VAR/local_hostname'
+        self._test_jinja_render_template(
+            fake_instance_data=fake_instance_data,
+            expected_result=expected_result)
+
+    def test_jinja_render_template_multiple_variables(self):
+        fake_instance_data = {
+            'v1': {
+                'localhostname': 'fake_hostname'
+            },
+            'ds': {
+                'meta_data': {
+                    'hostname': 'fake_hostname'
+                },
+                'meta-data': {
+                    'hostname': 'fake_hostname'
+                }
+            }
+        }
+        fake_template = b'{{ds.meta_data.hostname}}'
+        expected_result = b'fake_hostname'
+        self._test_jinja_render_template(
+            fake_instance_data=fake_instance_data,
+            expected_result=expected_result,
+            fake_template=fake_template)

cloudbaseinit/utils/template_engine/base_template.py

@@ -0,0 +1,60 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import re
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseTemplateEngine(object):
+    def __init__(self):
+        self._template_matcher = re.compile(r"##\s*template:(.*)", re.I)
+
+    @abc.abstractmethod
+    def get_template_type(self):
+        """Return the template type for the class loader"""
+        pass
+
+    @abc.abstractmethod
+    def render(self, data, template):
+        """Renders the template according to the data dictionary
+
+        The data variable is a dict which contains the key-values
+        that will be used to render the template.
+
+        The template is an encoded string which can contain special
+        constructions that will be used by the template engine.
+
+        The return value will be an encoded string.
+        """
+
+    def load(self, data):
+        """Returns True if the template header matches, False otherwise"""
+        template_type_matcher = self._template_matcher.match(data.decode())
+        template_type = template_type_matcher.group(1).lower().strip()
+        if self.get_template_type() == template_type:
+            return True
+
+    @staticmethod
+    def remove_template_definition(raw_template):
+        # Remove the first line, as it contains the template definition
+        template_split = raw_template.split(b"\n", 1)
+
+        if len(template_split) == 2:
+            # return the template without the header
+            return template_split[1]
+
+        # the template has just one line, return empty encoded string
+        return b''

cloudbaseinit/utils/template_engine/factory.py

@@ -0,0 +1,37 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from cloudbaseinit.utils import classloader
+from oslo_log import log as oslo_logging
+
+TEMPLATE_ENGINE_CLASS_PATHS = ["cloudbaseinit.utils.template_engine"
+                               ".jinja2_template.Jinja2TemplateEngine"]
+LOG = oslo_logging.getLogger(__name__)
+
+
+def get_template_engine(user_data):
+    """Returns the first template engine that loads correctly"""
+
+    cl = classloader.ClassLoader()
+    for class_path in TEMPLATE_ENGINE_CLASS_PATHS:
+        tpl_engine = cl.load_class(class_path)()
+        try:
+            if tpl_engine.load(user_data):
+                LOG.info("Using template engine: %s"
+                         % tpl_engine.get_template_type())
+                return tpl_engine
+        except Exception as ex:
+            LOG.error("Failed to load template engine '%s'" % class_path)
+            LOG.exception(ex)
+    return

cloudbaseinit/utils/template_engine/jinja2_template.py

@@ -0,0 +1,50 @@
+# Copyright 2019 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import jinja2
+
+from cloudbaseinit.utils.template_engine import base_template
+from jinja2 import runtime
+
+MISSING_JINJA_VARIABLE = u'CI_MISSING_JINJA_VAR/'
+
+
+@runtime.implements_to_string
+class MissingJinjaVariable(jinja2.DebugUndefined):
+    """Missing Jinja2 variable class."""
+
+    def __str__(self):
+        return u'%s%s' % (MISSING_JINJA_VARIABLE, self._undefined_name)
+
+
+class Jinja2TemplateEngine(base_template.BaseTemplateEngine):
+    def get_template_type(self):
+        return 'jinja'
+
+    def render(self, data, raw_template):
+        """Renders the template using Jinja2 template engine
+
+        The data variable is a dict which contains the key-values
+        that will be used to render the template.
+
+        The template is an encoded string which can contain special
+        constructions that will be used by the template engine.
+
+        The return value will be an encoded string.
+        """
+
+        template = self.remove_template_definition(raw_template).decode()
+        jinja_template = jinja2.Template(template,
+                                         trim_blocks=True,
+                                         undefined=MissingJinjaVariable,)
+        return jinja_template.render(**data).encode()

requirements.txt

@@ -12,6 +12,7 @@ netifaces
 PyYAML
 requests
 untangle==1.1.1
+jinja2
 pywin32;sys_platform=="win32"
 comtypes;sys_platform=="win32"
 pymi;sys_platform=="win32"

test-requirements.txt

@@ -10,3 +10,4 @@ stestr>=2.0.0
 openstackdocstheme>=1.11.0  # Apache-2.0
 # releasenotes
 reno>=1.8.0 # Apache-2.0
+ddt