Bring over the 'templater' from bzr

Change-Id: Iefcceb1c0d47ae455985f1d027465b0db0a0e467
This commit is contained in:
Joshua Harlow 2015-04-02 12:44:48 -07:00
parent bf5c513472
commit 92405e6712
4 changed files with 244 additions and 0 deletions

105
cloudinit/templater.py Normal file
View File

@ -0,0 +1,105 @@
# Copyright 2015 Canonical Ltd.
# This file is part of cloud-init. See LICENCE file for license information.
#
# vi: ts=4 expandtab
import collections
import logging
import os
import re
try:
import jinja2
from jinja2 import Template as JTemplate
JINJA_AVAILABLE = True
except (ImportError, AttributeError):
JINJA_AVAILABLE = False # noqa
LOG = logging.getLogger(__name__)
TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I)
BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
def basic_render(content, params):
"""This does simple replacement of bash variable like templates.
It identifies patterns like ${a} or $a and can also identify patterns like
${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
by key 'a'.
"""
def replacer(match):
# Only 1 of the 2 groups will actually have a valid entry.
name = match.group(1) or match.group(2)
if name is None:
# not sure how this can possibly occur
raise RuntimeError("Match encountered but no valid group present")
path = collections.deque(name.split("."))
selected_params = params
while len(path) > 1:
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError(
"Can not traverse into non-dictionary '%s' of type %s "
"while looking for subkey '%s'" %
(selected_params, type(selected_params), key))
selected_params = selected_params[key]
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError("Can not extract key '%s' from non-dictionary"
" '%s' of type %s"
% (key, selected_params, type(selected_params)))
return str(selected_params[key])
return BASIC_MATCHER.sub(replacer, content)
def detect_template(text):
def jinja_render(content, params):
# keep_trailing_newline is in jinja2 2.7+, not 2.6
add = "\n" if content.endswith("\n") else ""
return JTemplate(content,
undefined=jinja2.StrictUndefined,
trim_blocks=True).render(**params) + add
if "\n" in text:
ident, rest = text.split("\n", 1)
else:
ident = text
rest = ''
type_match = TYPE_MATCHER.match(ident)
if not type_match:
return ('basic', basic_render, text)
else:
template_type = type_match.group(1).lower().strip()
if template_type not in ('jinja', 'basic'):
raise ValueError("Unknown template rendering type '%s' requested"
% template_type)
if template_type == 'jinja' and not JINJA_AVAILABLE:
raise ValueError("Template requested jinja as renderer, but Jinja "
"is not available.")
elif template_type == 'jinja' and JINJA_AVAILABLE:
return ('jinja', jinja_render, rest)
# Only thing left over is the basic renderer (it is always available).
return ('basic', basic_render, rest)
def render_from_file(fn, params, encoding='utf-8'):
with open(fn, 'rb') as fh:
content = fh.read()
content = content.decode(encoding)
_, renderer, content = detect_template(content)
return renderer(content, params)
def render_to_file(fn, outfn, params, mode=0o644, encoding='utf-8'):
contents = render_from_file(fn, params, encoding=encoding)
with open(outfn, 'wb') as fh:
fh.write(contents.encode(encoding))
os.chmod(outfn, mode)
def render_string(content, params):
_, renderer, content = detect_template(content)
return renderer(content, params)

View File

@ -0,0 +1,137 @@
# Copyright 2015 Canonical Ltd.
# This file is part of cloud-init. See LICENCE file for license information.
#
# vi: ts=4 expandtab
import fixtures
import mock
import os
import textwrap
from cloudinit import templater
from cloudinit import test
class TestTemplates(test.TestCase):
jinja_tmpl = '\n'.join((
"## template:jinja",
"{{a}},{{b}}",
""
))
jinja_params = {'a': '1', 'b': '2'}
jinja_expected = '1,2\n'
def test_render_basic(self):
in_data = textwrap.dedent("""
${b}
c = d
""")
in_data = in_data.strip()
expected_data = textwrap.dedent("""
2
c = d
""")
out_data = templater.basic_render(in_data, {'b': 2})
self.assertEqual(expected_data.strip(), out_data)
def test_render_jinja(self):
c = templater.render_string(self.jinja_tmpl, self.jinja_params)
self.assertEqual(self.jinja_expected, c)
def test_render_jinja_crlf(self):
blob = '\r\n'.join((
"## template:jinja",
"{{a}},{{b}}"))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_default(self):
blob = '''$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_explict_default(self):
blob = '\n'.join(('## template: basic', '$a,$b',))
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEqual("1,2", c)
def test_render_basic_deeper(self):
hn = 'myfoohost.yahoo.com'
expected_data = "h=%s\nc=d\n" % hn
in_data = "h=$hostname.canonical_name\nc=d\n"
params = {
"hostname": {
"canonical_name": hn,
},
}
out_data = templater.render_string(in_data, params)
self.assertEqual(expected_data, out_data)
def test_render_basic_no_parens(self):
hn = "myfoohost"
in_data = "h=$hostname\nc=d\n"
expected_data = "h=%s\nc=d\n" % hn
out_data = templater.basic_render(in_data, {'hostname': hn})
self.assertEqual(expected_data, out_data)
def test_render_basic_parens(self):
hn = "myfoohost"
in_data = "h = ${hostname}\nc=d\n"
expected_data = "h = %s\nc=d\n" % hn
out_data = templater.basic_render(in_data, {'hostname': hn})
self.assertEqual(expected_data, out_data)
def test_render_basic2(self):
mirror = "mymirror"
codename = "zany"
in_data = "deb $mirror $codename-updates main contrib non-free"
ex_data = "deb %s %s-updates main contrib non-free" % (mirror,
codename)
out_data = templater.basic_render(
in_data, {'mirror': mirror, 'codename': codename})
self.assertEqual(ex_data, out_data)
def test_render_basic_exception_1(self):
in_data = "h=${foo.bar}"
self.assertRaises(
TypeError, templater.basic_render, in_data, {'foo': [1, 2]})
def test_unknown_renderer_raises_exception(self):
blob = '\n'.join((
"## template:bigfastcat",
"Hellow $name"
""))
self.assertRaises(
ValueError, templater.render_string, blob, {'name': 'foo'})
@mock.patch.object(templater, 'JINJA_AVAILABLE', False)
def test_jinja_without_jinja_raises_exception(self):
blob = '\n'.join((
"## template:jinja",
"Hellow {{name}}"
""))
templater.JINJA_AVAILABLE = False
self.assertRaises(
ValueError, templater.render_string, blob, {'name': 'foo'})
def test_render_from_file(self):
td = self.useFixture(fixtures.TempDir()).path
fname = os.path.join(td, "myfile")
with open(fname, "w") as fp:
fp.write(self.jinja_tmpl)
rendered = templater.render_from_file(fname, self.jinja_params)
self.assertEqual(rendered, self.jinja_expected)
def test_render_to_file(self):
td = self.useFixture(fixtures.TempDir()).path
src = os.path.join(td, "src")
target = os.path.join(td, "target")
with open(src, "w") as fp:
fp.write(self.jinja_tmpl)
templater.render_to_file(src, target, self.jinja_params)
with open(target, "r") as fp:
rendered = fp.read()
self.assertEqual(rendered, self.jinja_expected)

View File

@ -10,3 +10,4 @@ pyyaml
jsonpatch
requests>=1.0
requests-oauthlib
jinja2

View File

@ -6,6 +6,7 @@ httpretty>=0.7.1
mock
nose
testtools
fixtures
# For doc building
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3