diff --git a/fuel_ccp/common/jinja_utils.py b/fuel_ccp/common/jinja_utils.py index 87297107..9de669c1 100644 --- a/fuel_ccp/common/jinja_utils.py +++ b/fuel_ccp/common/jinja_utils.py @@ -1,4 +1,5 @@ import os +import re import socket import jinja2 @@ -41,3 +42,17 @@ def jinja_render(path, context, functions=(), ignore_undefined=False): env.globals[func.__name__] = func content = env.get_template(os.path.basename(path)).render(context) return content + + +def generate_jinja_imports(filenames): + """Generate str of jinja imports from list of filenames.""" + imports = [] # list of j2 imports: "{% import 'msg.j2' as msg %}" + for name in filenames: + import_as, extension = os.path.splitext(name) # remove file extension + if not re.match('[a-zA-Z_][a-zA-Z0-9_]*', import_as): + raise RuntimeError('Wrong templates file naming: the %s cannot be ' + 'imported by jinja with %s name. Please use ' + 'python compatible naming' % (name, import_as)) + imports.append( + "{% import '" + name + "' as " + import_as + " with context %}") + return ''.join(imports) diff --git a/fuel_ccp/common/utils.py b/fuel_ccp/common/utils.py index 75f1f78a..cd78027d 100644 --- a/fuel_ccp/common/utils.py +++ b/fuel_ccp/common/utils.py @@ -76,6 +76,27 @@ def address(service, port=None, external=False, with_scheme=False): return addr +def get_repositories_exports(repos_names=None): + """Load shared templates from ./export dirs of the repositories. """ + exports = dict() + repos_names = repos_names or [d['name'] for d in CONF.repositories.repos] + for repo in repos_names: + exports_dir = os.path.join(CONF.repositories.path, repo, 'exports') + if os.path.exists(exports_dir) and os.path.isdir(exports_dir): + for export in os.listdir(exports_dir): + path = os.path.join(exports_dir, export) + LOG.debug('Found shared jinja template file %s', path) + if export not in exports: + exports[export] = list() + with open(path) as f: + exports[export].append(f.read()) + + for export in exports: + exports[export] = '\n'.join(exports[export]) + + return exports + + def get_deploy_components_info(rendering_context=None): if rendering_context is None: rendering_context = CONF.configs._dict diff --git a/fuel_ccp/deploy.py b/fuel_ccp/deploy.py index ba74b660..8dc2613f 100644 --- a/fuel_ccp/deploy.py +++ b/fuel_ccp/deploy.py @@ -71,7 +71,7 @@ def process_files(files, service_dir): f["content"] = content -def parse_role(component, topology, configmaps): +def parse_role(component, topology, configmaps, jinja_imports): service_dir = component["service_dir"] role = component["service_content"] component_name = component["component_name"] @@ -82,7 +82,8 @@ def parse_role(component, topology, configmaps): _expand_files(service, role.get("files")) process_files(role.get("files"), service_dir) - files_cm = _create_files_configmap(service_name, role.get("files")) + files_cm = _create_files_configmap(service_name, role.get("files"), + jinja_imports) meta_cm = _create_meta_configmap(service) workflows = _parse_workflows(service) @@ -311,13 +312,13 @@ def _create_start_script_configmap(): return kubernetes.process_object(cm) -def _create_files_configmap(service_name, files): +def _create_files_configmap(service_name, files, macros_imports): configmap_name = "%s-%s" % (service_name, templates.FILES_CONFIG) data = {} if files: for filename, f in files.items(): with open(f["content"], "r") as f: - data[filename] = f.read() + data[filename] = macros_imports + f.read() data["placeholder"] = "" template = templates.serialize_configmap(configmap_name, data) return kubernetes.process_object(template) @@ -334,6 +335,13 @@ def _create_meta_configmap(service): return kubernetes.process_object(template) +def _create_jinja_templates_configmap(templates_files): + """Create config map of files from fuel-ccp-repo/exports dirs.""" + serialized = templates.serialize_configmap(templates.EXPORTS_CONFIG, + templates_files) + return kubernetes.process_object(serialized) + + def _make_topology(nodes, roles, replicas): failed = False # TODO(sreshetniak): move it to validation @@ -443,7 +451,8 @@ def check_images_change(objects): return False -def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology): +def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology, + jinja_imports): from_version = upgrade_data['_meta']['from'] to_version = upgrade_data['_meta']['to'] component = upgrade_data['_meta']['component'] @@ -457,7 +466,7 @@ def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology): step['files'] = {f: files[f] for f in step['files']} process_files(files, component['service_dir']) - _create_files_configmap(prefix, files) + _create_files_configmap(prefix, files, jinja_imports) container = { "name": prefix, "pre": [], @@ -541,14 +550,17 @@ def deploy_components(components_map, components): _create_namespace(CONF.configs) _create_globals_configmap(CONF.configs) start_script_cm = _create_start_script_configmap() - configmaps = (start_script_cm,) + + # Create cm with jinja config templates shared across all repositories. + templates_files = utils.get_repositories_exports() + jinja_imports = jinja_utils.generate_jinja_imports(templates_files.keys()) + templates_cm = _create_jinja_templates_configmap(templates_files) + configmaps = (start_script_cm, templates_cm) upgrading_components = {} for service_name in components: service = components_map[service_name] - objects_gen = parse_role(service, - topology=topology, - configmaps=configmaps) + objects_gen = parse_role(service, topology, configmaps, jinja_imports) objects = list(itertools.chain.from_iterable(objects_gen)) component_name = service['component_name'] do_upgrade = component_name in upgrading_components @@ -578,8 +590,9 @@ def deploy_components(components_map, components): upgrading_components[component_name][service_name] = objects for component_name, component_upg in upgrading_components.items(): - create_upgrade_jobs(component_name, component_upg, configmaps, - topology) + create_upgrade_jobs(component_name, component_upg, + configmaps, topology, + jinja_imports) if 'keystone' in components: _create_openrc(CONF.configs) diff --git a/fuel_ccp/templates.py b/fuel_ccp/templates.py index a6322370..eee63123 100644 --- a/fuel_ccp/templates.py +++ b/fuel_ccp/templates.py @@ -11,6 +11,7 @@ SCRIPT_CONFIG = "start-script" FILES_CONFIG = "files" META_CONFIG = "meta" ROLE_CONFIG = "role" +EXPORTS_CONFIG = "exports" ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py" PYTHON_PATH = "/usr/bin/python" @@ -68,6 +69,10 @@ def serialize_volume_mounts(container, for_job=None): "name": SCRIPT_CONFIG, "mountPath": "/opt/ccp_start_script/bin" }, + { + "name": EXPORTS_CONFIG, + "mountPath": "/etc/ccp/%s" % EXPORTS_CONFIG + }, { "name": FILES_CONFIG, "mountPath": "/etc/ccp/%s" % FILES_CONFIG @@ -250,10 +255,16 @@ def serialize_volumes(service, for_job=None): "name": "%s-%s" % (service["name"], FILES_CONFIG), "items": file_items } + }, + { + "name": EXPORTS_CONFIG, + "configMap": { + "name": EXPORTS_CONFIG, + } } ] volume_names = [GLOBAL_CONFIG, META_CONFIG, ROLE_CONFIG, SCRIPT_CONFIG, - FILES_CONFIG] + FILES_CONFIG, EXPORTS_CONFIG] for cont in itertools.chain(service["containers"], [for_job]): for v in cont.get("volumes", ()): if v["name"] in volume_names: diff --git a/fuel_ccp/tests/test_templates.py b/fuel_ccp/tests/test_templates.py index 7f536a93..e6c9fac6 100644 --- a/fuel_ccp/tests/test_templates.py +++ b/fuel_ccp/tests/test_templates.py @@ -44,6 +44,7 @@ class TestDeploy(base.TestCase): {'mountPath': '/etc/ccp/meta', 'name': 'meta'}, {'mountPath': '/opt/ccp_start_script/bin', 'name': 'start-script'}, + {'mountPath': '/etc/ccp/exports', 'name': 'exports'}, {'mountPath': '/etc/ccp/files', 'name': 'files'} ], "readinessProbe": {