diff --git a/cloudbaseinit/plugins/common/userdata.py b/cloudbaseinit/plugins/common/userdata.py index 4b06047b..be251743 100644 --- a/cloudbaseinit/plugins/common/userdata.py +++ b/cloudbaseinit/plugins/common/userdata.py @@ -26,6 +26,7 @@ from cloudbaseinit.plugins.common import execcmd from cloudbaseinit.plugins.common.userdataplugins import factory from cloudbaseinit.plugins.common import userdatautils from cloudbaseinit.utils import encoding +from cloudbaseinit.utils.template_engine import factory as template_factory from cloudbaseinit.utils import x509constants @@ -53,7 +54,7 @@ class UserDataPlugin(base.BasePlugin): self._write_userdata(user_data, user_data_path) if CONF.process_userdata: - return self._process_user_data(user_data) + return self._process_user_data(user_data, service) return base.PLUGIN_EXECUTION_DONE, False @staticmethod @@ -100,7 +101,7 @@ class UserDataPlugin(base.BasePlugin): "The user data content is " "either invalid or empty.") - def _process_user_data(self, user_data): + def _process_user_data(self, user_data, service): plugin_status = base.PLUGIN_EXECUTION_DONE reboot = False headers = self._get_headers(user_data) @@ -122,7 +123,7 @@ class UserDataPlugin(base.BasePlugin): return plugin_status, reboot else: - return self._process_non_multi_part(user_data) + return self._process_non_multi_part(user_data, service) def _process_part(self, part, user_data_plugins, user_handlers): ret_val = None @@ -186,8 +187,16 @@ class UserDataPlugin(base.BasePlugin): LOG.debug("Calling part handler \"__end__\" event") handler_func(None, "__end__", None, None) - def _process_non_multi_part(self, user_data): + def _process_non_multi_part(self, user_data, service): ret_val = None + + template_engine = template_factory.get_template_engine(user_data) + if template_engine: + user_data = template_engine.render( + service.get_instance_data(), + user_data + ) + if user_data.startswith(b'#cloud-config'): user_data_plugins = factory.load_plugins() cloud_config_plugin = user_data_plugins.get('text/cloud-config') diff --git a/cloudbaseinit/tests/plugins/common/test_userdata.py b/cloudbaseinit/tests/plugins/common/test_userdata.py index 7db860d1..c734b908 100644 --- a/cloudbaseinit/tests/plugins/common/test_userdata.py +++ b/cloudbaseinit/tests/plugins/common/test_userdata.py @@ -162,8 +162,10 @@ class UserDataPluginTest(unittest.TestCase): mock_part = mock.MagicMock() mock_parse_mime.return_value = [mock_part] mock_process_part.return_value = (base.PLUGIN_EXECUTION_DONE, reboot) + mock_service = mock.MagicMock() - response = self._userdata._process_user_data(user_data=user_data) + response = self._userdata._process_user_data(user_data=user_data, + service=mock_service) if user_data.startswith(b'Content-Type: multipart'): mock_load_plugins.assert_called_once_with() @@ -172,7 +174,8 @@ class UserDataPluginTest(unittest.TestCase): mock_load_plugins(), {}) self.assertEqual((base.PLUGIN_EXECUTION_DONE, reboot), response) else: - mock_process_non_multi_part.assert_called_once_with(user_data) + mock_process_non_multi_part.assert_called_once_with(user_data, + mock_service) self.assertEqual(mock_process_non_multi_part.return_value, response) @@ -313,8 +316,9 @@ class UserDataPluginTest(unittest.TestCase): '.execute_user_data_script') def test_process_non_multi_part(self, mock_execute_user_data_script): user_data = b'fake' + service = mock.MagicMock() status, reboot = self._userdata._process_non_multi_part( - user_data=user_data) + user_data=user_data, service=service) mock_execute_user_data_script.assert_called_once_with(user_data) self.assertEqual(status, 1) self.assertFalse(reboot) @@ -329,10 +333,11 @@ class UserDataPluginTest(unittest.TestCase): b2NhbGhvc3QwHhcNMTUwNjE1MTAyODUxWhcNMjUwNjEyMTAyODUxWjAbMRkwFwYD -----END CERTIFICATE----- ''').encode() + service = mock.MagicMock() with testutils.LogSnatcher('cloudbaseinit.plugins.' 'common.userdata') as snatcher: status, reboot = self._userdata._process_non_multi_part( - user_data=user_data) + user_data=user_data, service=service) expected_logging = ['Found X509 certificate in userdata'] self.assertFalse(mock_execute_user_data_script.called) @@ -340,25 +345,54 @@ class UserDataPluginTest(unittest.TestCase): self.assertEqual(1, status) self.assertFalse(reboot) + @mock.patch('cloudbaseinit.utils.template_engine.factory.' + 'get_template_engine') @mock.patch('cloudbaseinit.plugins.common.userdataplugins.factory.' 'load_plugins') - def test_process_non_multi_part_cloud_config(self, mock_load_plugins): - user_data = b'#cloud-config' + def _test_process_non_multi_part_cloud_config(self, mock_load_plugins, + mock_load_templates, + user_data, + expected_userdata, + template_renderer=None): + mock_service = mock.MagicMock() mock_return_value = mock.sentinel.return_value mock_cloud_config_plugin = mock.Mock() mock_cloud_config_plugin.process.return_value = mock_return_value mock_load_plugins.return_value = { 'text/cloud-config': mock_cloud_config_plugin} + mock_load_templates.return_value = template_renderer status, reboot = self._userdata._process_non_multi_part( - user_data=user_data) + user_data=user_data, service=mock_service) + + if template_renderer: + mock_load_plugins.assert_called_once_with() + + (mock_cloud_config_plugin + .process_non_multipart + .assert_called_once_with(expected_userdata)) - mock_load_plugins.assert_called_once_with() - (mock_cloud_config_plugin - .process_non_multipart - .assert_called_once_with(user_data)) self.assertEqual(status, 1) self.assertFalse(reboot) + def test_process_non_multi_part_cloud_config(self): + user_data = b'#cloud-config' + self._test_process_non_multi_part_cloud_config( + user_data=user_data, expected_userdata=user_data) + + def test_process_non_multi_part_cloud_config_jinja(self): + user_data = b'## template:jinja\n#cloud-config' + expected_userdata = b'#cloud-config' + mock_template_renderer = mock.MagicMock() + mock_template_renderer.render.return_value = expected_userdata + self._test_process_non_multi_part_cloud_config( + user_data=user_data, expected_userdata=expected_userdata, + template_renderer=mock_template_renderer) + + def test_process_non_multi_part_no_valid_template(self): + user_data = b'## template:none' + self._test_process_non_multi_part_cloud_config( + user_data=user_data, expected_userdata=user_data) + class TestCloudConfig(unittest.TestCase): @classmethod diff --git a/cloudbaseinit/tests/utils/template_engine/__init__.py b/cloudbaseinit/tests/utils/template_engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudbaseinit/tests/utils/template_engine/test_base_template.py b/cloudbaseinit/tests/utils/template_engine/test_base_template.py new file mode 100644 index 00000000..10717cbd --- /dev/null +++ b/cloudbaseinit/tests/utils/template_engine/test_base_template.py @@ -0,0 +1,30 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import unittest + +from cloudbaseinit.utils.template_engine import base_template as bt + + +@ddt.ddt +class TestBaseTemplateEngine(unittest.TestCase): + + @ddt.data((b'', b''), + (b'## template:jinja test', b''), + (b'## template:jinja \ntest', b'test')) + @ddt.unpack + def test_remove_template_definition(self, template, expected_output): + output = bt.BaseTemplateEngine.remove_template_definition(template) + self.assertEqual(expected_output, output) diff --git a/cloudbaseinit/tests/utils/template_engine/test_factory.py b/cloudbaseinit/tests/utils/template_engine/test_factory.py new file mode 100644 index 00000000..a7f5868e --- /dev/null +++ b/cloudbaseinit/tests/utils/template_engine/test_factory.py @@ -0,0 +1,58 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + import unittest.mock as mock +except ImportError: + import mock +import unittest + +from cloudbaseinit.utils.template_engine import factory + + +class FakeLoaderError(Exception): + pass + + +class TestTemplateFactory(unittest.TestCase): + + def test_get_template_engine_empty(self): + fake_userdata = b'' + result = factory.get_template_engine(fake_userdata) + self.assertEqual(result, None) + + def test_get_template_engine_no_match(self): + fake_userdata = b'no match' + result = factory.get_template_engine(fake_userdata) + self.assertEqual(result, None) + + def test_get_template_engine_not_supported(self): + fake_userdata = b'## template:fake' + result = factory.get_template_engine(fake_userdata) + self.assertEqual(result, None) + + @mock.patch('cloudbaseinit.utils.classloader.ClassLoader') + def test_get_template_engine(self, mock_class_loader): + fake_userdata = b'## template:jinja' + mock_load_class = mock_class_loader.return_value.load_class + self.assertEqual(mock_load_class.return_value.return_value, + factory.get_template_engine(fake_userdata)) + + @mock.patch('cloudbaseinit.utils.classloader.ClassLoader') + def test_get_template_engine_class_not_found(self, mock_class_loader): + fake_userdata = b'## template:jinja' + mock_class_loader.return_value.load_class.side_effect = ( + FakeLoaderError) + self.assertRaises(FakeLoaderError, + factory.get_template_engine, fake_userdata) diff --git a/cloudbaseinit/tests/utils/template_engine/test_jinja2_template.py b/cloudbaseinit/tests/utils/template_engine/test_jinja2_template.py new file mode 100644 index 00000000..5d9f52a7 --- /dev/null +++ b/cloudbaseinit/tests/utils/template_engine/test_jinja2_template.py @@ -0,0 +1,82 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import unittest +try: + import unittest.mock as mock +except ImportError: + import mock + +from cloudbaseinit.utils.template_engine.jinja2_template import ( + Jinja2TemplateEngine) + + +class TestJinja2TemplateEngine(unittest.TestCase): + + @mock.patch('cloudbaseinit.utils.template_engine.base_template' + '.BaseTemplateEngine.remove_template_definition') + def _test_jinja_render_template(self, mock_remove_header, + fake_instance_data, expected_result, + fake_template = b'{{v1.local_hostname}}'): + + mock_remove_header.return_value = fake_template + + output = Jinja2TemplateEngine().render(fake_instance_data, + fake_template) + + self.assertEqual(expected_result, output) + + def test_jinja_render_template(self): + fake_instance_data = { + 'v1': { + 'local_hostname': 'fake_hostname' + } + } + expected_result = b'fake_hostname' + self._test_jinja_render_template( + fake_instance_data=fake_instance_data, + expected_result=expected_result) + + def test_jinja_render_template_missing_variable(self): + fake_instance_data = { + 'v1': { + 'localhostname': 'fake_hostname' + } + } + expected_result = b'CI_MISSING_JINJA_VAR/local_hostname' + self._test_jinja_render_template( + fake_instance_data=fake_instance_data, + expected_result=expected_result) + + def test_jinja_render_template_multiple_variables(self): + fake_instance_data = { + 'v1': { + 'localhostname': 'fake_hostname' + }, + 'ds': { + 'meta_data': { + 'hostname': 'fake_hostname' + }, + 'meta-data': { + 'hostname': 'fake_hostname' + } + } + } + fake_template = b'{{ds.meta_data.hostname}}' + expected_result = b'fake_hostname' + self._test_jinja_render_template( + fake_instance_data=fake_instance_data, + expected_result=expected_result, + fake_template=fake_template) diff --git a/cloudbaseinit/utils/template_engine/__init__.py b/cloudbaseinit/utils/template_engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudbaseinit/utils/template_engine/base_template.py b/cloudbaseinit/utils/template_engine/base_template.py new file mode 100644 index 00000000..d3ae7aaa --- /dev/null +++ b/cloudbaseinit/utils/template_engine/base_template.py @@ -0,0 +1,60 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import re +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseTemplateEngine(object): + def __init__(self): + self._template_matcher = re.compile(r"##\s*template:(.*)", re.I) + + @abc.abstractmethod + def get_template_type(self): + """Return the template type for the class loader""" + pass + + @abc.abstractmethod + def render(self, data, template): + """Renders the template according to the data dictionary + + The data variable is a dict which contains the key-values + that will be used to render the template. + + The template is an encoded string which can contain special + constructions that will be used by the template engine. + + The return value will be an encoded string. + """ + + def load(self, data): + """Returns True if the template header matches, False otherwise""" + template_type_matcher = self._template_matcher.match(data.decode()) + template_type = template_type_matcher.group(1).lower().strip() + if self.get_template_type() == template_type: + return True + + @staticmethod + def remove_template_definition(raw_template): + # Remove the first line, as it contains the template definition + template_split = raw_template.split(b"\n", 1) + + if len(template_split) == 2: + # return the template without the header + return template_split[1] + + # the template has just one line, return empty encoded string + return b'' diff --git a/cloudbaseinit/utils/template_engine/factory.py b/cloudbaseinit/utils/template_engine/factory.py new file mode 100644 index 00000000..68ef224e --- /dev/null +++ b/cloudbaseinit/utils/template_engine/factory.py @@ -0,0 +1,37 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cloudbaseinit.utils import classloader +from oslo_log import log as oslo_logging + +TEMPLATE_ENGINE_CLASS_PATHS = ["cloudbaseinit.utils.template_engine" + ".jinja2_template.Jinja2TemplateEngine"] +LOG = oslo_logging.getLogger(__name__) + + +def get_template_engine(user_data): + """Returns the first template engine that loads correctly""" + + cl = classloader.ClassLoader() + for class_path in TEMPLATE_ENGINE_CLASS_PATHS: + tpl_engine = cl.load_class(class_path)() + try: + if tpl_engine.load(user_data): + LOG.info("Using template engine: %s" + % tpl_engine.get_template_type()) + return tpl_engine + except Exception as ex: + LOG.error("Failed to load template engine '%s'" % class_path) + LOG.exception(ex) + return diff --git a/cloudbaseinit/utils/template_engine/jinja2_template.py b/cloudbaseinit/utils/template_engine/jinja2_template.py new file mode 100644 index 00000000..c5925309 --- /dev/null +++ b/cloudbaseinit/utils/template_engine/jinja2_template.py @@ -0,0 +1,50 @@ +# Copyright 2019 Cloudbase Solutions Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import jinja2 + +from cloudbaseinit.utils.template_engine import base_template +from jinja2 import runtime + +MISSING_JINJA_VARIABLE = u'CI_MISSING_JINJA_VAR/' + + +@runtime.implements_to_string +class MissingJinjaVariable(jinja2.DebugUndefined): + """Missing Jinja2 variable class.""" + + def __str__(self): + return u'%s%s' % (MISSING_JINJA_VARIABLE, self._undefined_name) + + +class Jinja2TemplateEngine(base_template.BaseTemplateEngine): + def get_template_type(self): + return 'jinja' + + def render(self, data, raw_template): + """Renders the template using Jinja2 template engine + + The data variable is a dict which contains the key-values + that will be used to render the template. + + The template is an encoded string which can contain special + constructions that will be used by the template engine. + + The return value will be an encoded string. + """ + + template = self.remove_template_definition(raw_template).decode() + jinja_template = jinja2.Template(template, + trim_blocks=True, + undefined=MissingJinjaVariable,) + return jinja_template.render(**data).encode() diff --git a/requirements.txt b/requirements.txt index 02ff0e48..f977d627 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ netifaces PyYAML requests untangle==1.1.1 +jinja2 pywin32;sys_platform=="win32" comtypes;sys_platform=="win32" pymi;sys_platform=="win32" diff --git a/test-requirements.txt b/test-requirements.txt index e6c5a737..59da2aef 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,3 +10,4 @@ stestr>=2.0.0 openstackdocstheme>=1.11.0 # Apache-2.0 # releasenotes reno>=1.8.0 # Apache-2.0 +ddt