x
/
fuel-ccp
Code Issues Proposed changes

Merge "Introduce cross-repository config templating"

This commit is contained in:
Jenkins 2016-12-14 13:23:09 +00:00 committed by Gerrit Code Review
parent 771e0fb029 29133f2787
commit 074fbf089e
5 changed files with 74 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
fuel_ccp/common/jinja_utils.py
View File

@ -1,4 +1,5 @@
import os
import re
import socket
import jinja2
@ -41,3 +42,17 @@ def jinja_render(path, context, functions=(), ignore_undefined=False):
env.globals[func.__name__] = func
content = env.get_template(os.path.basename(path)).render(context)
return content
def generate_jinja_imports(filenames):
"""Generate str of jinja imports from list of filenames."""
imports = [] # list of j2 imports: "{% import 'msg.j2' as msg %}"
for name in filenames:
import_as, extension = os.path.splitext(name) # remove file extension
if not re.match('[a-zA-Z_][a-zA-Z0-9_]*', import_as):
raise RuntimeError('Wrong templates file naming: the %s cannot be '
'imported by jinja with %s name. Please use '
'python compatible naming' % (name, import_as))
imports.append(
"{% import '" + name + "' as " + import_as + " with context %}")
return ''.join(imports)

21
fuel_ccp/common/utils.py
View File

@ -76,6 +76,27 @@ def address(service, port=None, external=False, with_scheme=False):
return addr
def get_repositories_exports(repos_names=None):
"""Load shared templates from ./export dirs of the repositories. """
exports = dict()
repos_names = repos_names or [d['name'] for d in CONF.repositories.repos]
for repo in repos_names:
exports_dir = os.path.join(CONF.repositories.path, repo, 'exports')
if os.path.exists(exports_dir) and os.path.isdir(exports_dir):
for export in os.listdir(exports_dir):
path = os.path.join(exports_dir, export)
LOG.debug('Found shared jinja template file %s', path)
if export not in exports:
exports[export] = list()
with open(path) as f:
exports[export].append(f.read())
for export in exports:
exports[export] = '\n'.join(exports[export])
return exports
def get_deploy_components_info(rendering_context=None):
if rendering_context is None:
rendering_context = CONF.configs._dict

37
fuel_ccp/deploy.py
View File

@ -71,7 +71,7 @@ def process_files(files, service_dir):
f["content"] = content
def parse_role(component, topology, configmaps):
def parse_role(component, topology, configmaps, jinja_imports):
service_dir = component["service_dir"]
role = component["service_content"]
component_name = component["component_name"]
@ -82,7 +82,8 @@ def parse_role(component, topology, configmaps):
_expand_files(service, role.get("files"))
process_files(role.get("files"), service_dir)
files_cm = _create_files_configmap(service_name, role.get("files"))
files_cm = _create_files_configmap(service_name, role.get("files"),
jinja_imports)
meta_cm = _create_meta_configmap(service)
workflows = _parse_workflows(service)
@ -311,13 +312,13 @@ def _create_start_script_configmap():
return kubernetes.process_object(cm)
def _create_files_configmap(service_name, files):
def _create_files_configmap(service_name, files, macros_imports):
configmap_name = "%s-%s" % (service_name, templates.FILES_CONFIG)
data = {}
if files:
for filename, f in files.items():
with open(f["content"], "r") as f:
data[filename] = f.read()
data[filename] = macros_imports + f.read()
data["placeholder"] = ""
template = templates.serialize_configmap(configmap_name, data)
return kubernetes.process_object(template)
@ -334,6 +335,13 @@ def _create_meta_configmap(service):
return kubernetes.process_object(template)
def _create_jinja_templates_configmap(templates_files):
"""Create config map of files from fuel-ccp-repo/exports dirs."""
serialized = templates.serialize_configmap(templates.EXPORTS_CONFIG,
templates_files)
return kubernetes.process_object(serialized)
def _make_topology(nodes, roles, replicas):
failed = False
# TODO(sreshetniak): move it to validation
@ -443,7 +451,8 @@ def check_images_change(objects):
return False
def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology):
def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology,
jinja_imports):
from_version = upgrade_data['_meta']['from']
to_version = upgrade_data['_meta']['to']
component = upgrade_data['_meta']['component']
@ -457,7 +466,7 @@ def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology):
step['files'] = {f: files[f] for f in step['files']}
process_files(files, component['service_dir'])
_create_files_configmap(prefix, files)
_create_files_configmap(prefix, files, jinja_imports)
container = {
"name": prefix,
"pre": [],
@ -541,14 +550,17 @@ def deploy_components(components_map, components):
_create_namespace(CONF.configs)
_create_globals_configmap(CONF.configs)
start_script_cm = _create_start_script_configmap()
configmaps = (start_script_cm,)
# Create cm with jinja config templates shared across all repositories.
templates_files = utils.get_repositories_exports()
jinja_imports = jinja_utils.generate_jinja_imports(templates_files.keys())
templates_cm = _create_jinja_templates_configmap(templates_files)
configmaps = (start_script_cm, templates_cm)
upgrading_components = {}
for service_name in components:
service = components_map[service_name]
objects_gen = parse_role(service,
topology=topology,
configmaps=configmaps)
objects_gen = parse_role(service, topology, configmaps, jinja_imports)
objects = list(itertools.chain.from_iterable(objects_gen))
component_name = service['component_name']
do_upgrade = component_name in upgrading_components
@ -578,8 +590,9 @@ def deploy_components(components_map, components):
upgrading_components[component_name][service_name] = objects
for component_name, component_upg in upgrading_components.items():
create_upgrade_jobs(component_name, component_upg, configmaps,
topology)
create_upgrade_jobs(component_name, component_upg,
configmaps, topology,
jinja_imports)
if 'keystone' in components:
_create_openrc(CONF.configs)

13
fuel_ccp/templates.py
View File

@ -11,6 +11,7 @@ SCRIPT_CONFIG = "start-script"
FILES_CONFIG = "files"
META_CONFIG = "meta"
ROLE_CONFIG = "role"
EXPORTS_CONFIG = "exports"
ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py"
PYTHON_PATH = "/usr/bin/python"
@ -68,6 +69,10 @@ def serialize_volume_mounts(container, for_job=None):
"name": SCRIPT_CONFIG,
"mountPath": "/opt/ccp_start_script/bin"
},
{
"name": EXPORTS_CONFIG,
"mountPath": "/etc/ccp/%s" % EXPORTS_CONFIG
},
{
"name": FILES_CONFIG,
"mountPath": "/etc/ccp/%s" % FILES_CONFIG
@ -250,10 +255,16 @@ def serialize_volumes(service, for_job=None):
"name": "%s-%s" % (service["name"], FILES_CONFIG),
"items": file_items
}
},
{
"name": EXPORTS_CONFIG,
"configMap": {
"name": EXPORTS_CONFIG,
}
}
]
volume_names = [GLOBAL_CONFIG, META_CONFIG, ROLE_CONFIG, SCRIPT_CONFIG,
FILES_CONFIG]
FILES_CONFIG, EXPORTS_CONFIG]
for cont in itertools.chain(service["containers"], [for_job]):
for v in cont.get("volumes", ()):
if v["name"] in volume_names:

1
fuel_ccp/tests/test_templates.py
View File

@ -44,6 +44,7 @@ class TestDeploy(base.TestCase):
{'mountPath': '/etc/ccp/meta', 'name': 'meta'},
{'mountPath': '/opt/ccp_start_script/bin',
'name': 'start-script'},
{'mountPath': '/etc/ccp/exports', 'name': 'exports'},
{'mountPath': '/etc/ccp/files', 'name': 'files'}
],
"readinessProbe": {