diff --git a/fuel_ccp/action.py b/fuel_ccp/action.py index ace503f7..40989081 100644 --- a/fuel_ccp/action.py +++ b/fuel_ccp/action.py @@ -56,10 +56,19 @@ class Action(object): self.user_parameters = user_parameters else: self.user_parameters = () + self._process_dependencies() self._create_configmap() self._create_action() return self.k8s_name + def _process_dependencies(self): + services_map = utils.get_deploy_components_info() + deps_map = utils.get_dependencies_map(services_map) + new_deps = [] + for dep in self.dependencies: + new_deps.extend(utils.extend_dependency(dep, deps_map, {}, {})) + self.dependencies = new_deps + # configmap methods def _create_configmap(self): diff --git a/fuel_ccp/cleanup.py b/fuel_ccp/cleanup.py index b8e0a573..1d8840b1 100644 --- a/fuel_ccp/cleanup.py +++ b/fuel_ccp/cleanup.py @@ -131,7 +131,7 @@ def _cleanup_openstack_environment(configs, auth_url=None, verify=True): 'is not deployed') configs['auth_url'] = auth_url or '%s/v3' % utils.address( - 'keystone', configs['keystone']['public_port'], True, True) + {}, 'keystone', configs['keystone']['public_port'], True, True) session = _get_session( configs['auth_url'], configs['openstack']['user_name'], diff --git a/fuel_ccp/common/utils.py b/fuel_ccp/common/utils.py index beacf660..13890f32 100644 --- a/fuel_ccp/common/utils.py +++ b/fuel_ccp/common/utils.py @@ -3,6 +3,7 @@ import logging import os import pkg_resources +import jinja2 import yaml import fuel_ccp @@ -60,7 +61,8 @@ def get_config_paths(): return paths -def address(service, port=None, external=False, with_scheme=False): +@jinja2.contextfunction +def address(ctx, service, port=None, external=False, with_scheme=False): addr = None service_name = service.split('-')[0] enable_tls = CONF.configs.get(service_name, {}).get( @@ -81,6 +83,15 @@ def address(service, port=None, external=False, with_scheme=False): elif port.get('node'): addr = '%s:%s' % (CONF.configs.k8s_external_ip, port['node']) + current_service = ctx.get('_current_service') + if current_service: + current_service_def = CONF.services.get(current_service, {}).get( + 'service_def') + if current_service_def == service: + service = current_service + else: + service = CONF.services.get(current_service, {}).get( + 'mapping', {}).get(service) or service if addr is None: addr = '.'.join((service, CONF.kubernetes.namespace, 'svc', CONF.kubernetes.cluster_domain)) @@ -122,10 +133,60 @@ def get_component_name_from_repo_path(path): return name +def get_service_definitions_map(): + """Maps each service definition to its custom services""" + s_d_map = {} + for service_name, value in CONF.services._items(): + s_d_map.setdefault(value['service_def'], []) + s_d_map[value['service_def']].append(service_name) + return s_d_map + + +def extend_dependency(dep, deps_map, services_map, service_mapping): + """Extends dependencies with service prefix""" + dep_name = dep.split(':')[0] + if dep_name not in deps_map: + # dependency is not a container or job + # checking service mapping first + if dep_name in service_mapping: + dep_name = service_mapping[dep_name] + service_ref = services_map[dep_name] + elif dep_name in services_map: + service_ref = services_map[dep_name] + else: + raise RuntimeError('Dependency "%s" not found' % dep_name) + # adjust deps with container names of the service + return ["%s/%s" % (dep_name, cnt['name']) for cnt in service_ref[ + 'service_content']['service']['containers']] + + dep_service_def = deps_map[dep_name] + dep_service_name = service_mapping.get( + dep_service_def) or dep_service_def + return ["%s/%s" % (dep_service_name, dep)] + + +def process_dependencies(service, deps_map, services_map): + service_name = service['service_content']['service']['name'] + service_mapping = CONF.services.get(service_name, {}).get('mapping', {}) + containers = service['service_content']['service']['containers'] + for cont in containers: + for cmd in itertools.chain( + cont.get('pre', []), [cont.get('daemon', [])], + cont.get('post', [])): + if cmd.get('dependencies'): + new_deps = [] + for dep in cmd['dependencies']: + new_deps.extend(extend_dependency( + dep, deps_map, services_map, service_mapping)) + cmd['dependencies'] = new_deps + + def get_deploy_components_info(rendering_context=None): if rendering_context is None: rendering_context = CONF.configs._dict - components_map = {} + service_definitions_map = get_service_definitions_map() + services_map = {} + custom_services_map = {} for repo in get_repositories_paths(): service_dir = os.path.join(repo, "service") @@ -160,18 +221,44 @@ def get_deploy_components_info(rendering_context=None): LOG.debug("Parse service definition: %s", service_file) service_definition = yaml.load(content) service_name = service_definition['service']['name'] - components_map[service_name] = { + services_map[service_name] = { 'component': component, 'component_name': component_name, 'service_dir': service_dir, 'service_content': service_definition } - return components_map + for svc in service_definitions_map.get(service_name, ()): + LOG.debug("Rendering service definition: %s for '%s' " + "service", service_file, svc) + context = rendering_context.copy() + context['_current_service'] = svc + content = jinja_utils.jinja_render( + os.path.join(service_dir, service_file), + context, functions=[address] + ) + LOG.debug("Parse service definition: %s for '%s' " + "service", service_file, svc) + service_definition = yaml.load(content) + service_definition['service']['name'] = svc + custom_services_map[svc] = { + 'component': component, + 'component_name': component_name, + 'service_dir': service_dir, + 'service_content': service_definition + } + + deps_map = get_dependencies_map(services_map) + services_map.update(custom_services_map) + for svc_name, svc in services_map.items(): + process_dependencies(svc, deps_map, services_map) + + return services_map -def get_dependencies_map(components_map): +def get_dependencies_map(services_map): + """Maps each container and job to its service""" deps_map = {} - for service_name, service in components_map.items(): + for service_name, service in services_map.items(): containers = service['service_content']['service']['containers'] for cont in containers: deps_map[cont['name']] = service_name diff --git a/fuel_ccp/config/__init__.py b/fuel_ccp/config/__init__.py index e5e89026..9af847d1 100644 --- a/fuel_ccp/config/__init__.py +++ b/fuel_ccp/config/__init__.py @@ -12,6 +12,7 @@ from fuel_ccp.config import kubernetes from fuel_ccp.config import registry from fuel_ccp.config import replicas from fuel_ccp.config import repositories +from fuel_ccp.config import services from fuel_ccp.config import sources from fuel_ccp.config import url @@ -56,7 +57,7 @@ CONF = _Wrapper() CONFIG_MODULES = [ builder, cli, images, kubernetes, registry, replicas, repositories, - sources, url, files, + sources, url, files, services, ] @@ -117,7 +118,7 @@ def load_component_defaults(): from fuel_ccp.common import utils sections = ['versions', 'sources', 'configs', 'nodes', 'roles', 'replicas', - 'url', 'files'] + 'url', 'files', 'services'] new_config = _yaml.AttrDict((k, _yaml.AttrDict()) for k in sections) for path in utils.get_config_paths(): if not os.path.exists(path): @@ -134,6 +135,7 @@ def load_component_defaults(): new_config['configs']['namespace'] = _REAL_CONF.kubernetes.namespace new_config['configs'][ 'cluster_domain'] = _REAL_CONF.kubernetes.cluster_domain + new_config['configs']['services'] = _REAL_CONF.services new_config._merge(_REAL_CONF) # FIXME workaround to not deep merge 'sources' config for k, v in _REAL_CONF.sources._items(): diff --git a/fuel_ccp/config/services.py b/fuel_ccp/config/services.py new file mode 100644 index 00000000..bfd1d72d --- /dev/null +++ b/fuel_ccp/config/services.py @@ -0,0 +1,17 @@ +SCHEMA = { + "services": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": False, + "required": ["service_def"], + "properties": { + "service_def": {"type": "string"}, + "mapping": {"type": "object"}, + } + } + } +} +DEFAULTS = { + "services": {}, +} diff --git a/fuel_ccp/dependencies.py b/fuel_ccp/dependencies.py index a498a49b..f708e970 100644 --- a/fuel_ccp/dependencies.py +++ b/fuel_ccp/dependencies.py @@ -12,147 +12,48 @@ Example: ccp --config-file=~/ccp.conf show-dep nova-api nova-compute """ +import itertools import logging -import re -import sys from fuel_ccp.common import utils from fuel_ccp.validation import base as base_validation LOG = logging.getLogger(__name__) -YAML_FILE_RE = re.compile(r'\.yaml$') + +def get_service_deps(graph, service): + visited, stack = set(), [service] + while stack: + vertex = stack.pop() + if vertex not in visited: + visited.add(vertex) + stack.extend(graph[vertex] - visited) + return visited -class Node(object): - """Reperesents dependency. Service or job.""" - - def __init__(self, name, sort, dependencies=None, job_parent=None): - self.name = name - self.sort = sort - if sort not in ['service', 'job']: - msg = "'sort' attribute must be 'service' or 'job' not \ - '{sort}'".format(sort=sort) - raise ValueError(msg) - - self.dependencies = dependencies or [] - self.job_parent = job_parent - - if self.sort == 'job' and self.job_parent is None: - msg = "'job_parent' attribute for 'job' mustn't be None" - raise ValueError(msg) - - def is_service(self): - return self.sort == 'service' - - -def get_deps_map(components_map=None): +def get_deps_graph(components_map=None): """Returns dependencies map.""" components_map = components_map or utils.get_deploy_components_info() - - deps_map = {} - for service_name, service_map in components_map.items(): - deps_map[service_name] = Node( - service_name, 'service', _parse_service_deps( - service_map['service_content'])) - deps_map.update(_parse_pre_and_post_deps( - service_map['service_content'])) - - return deps_map + deps_graph = {} + for service_name, service in components_map.items(): + deps_graph[service_name] = set() + containers = service['service_content']['service']['containers'] + for cont in containers: + for cmd in itertools.chain( + cont.get('pre', []), [cont.get('daemon', [])], + cont.get('post', [])): + for dep in cmd.get('dependencies', ()): + deps_graph[service_name].add(dep.partition("/")[0]) + return deps_graph -def _prepare_deps(deps): - return [dep.partition(":")[0] for dep in deps] - - -def _parse_service_deps(service_map): - """Parses service map and finds dependencies of daemons.""" +def get_deps(components, components_map): + deps_graph = get_deps_graph(components_map) dependencies = set() - for container in service_map['service']['containers']: - cont_deps = container['daemon'].get('dependencies', []) - dependencies.update(_prepare_deps(cont_deps)) - for pre in container.get('pre', []): - if pre.get('type') == 'single': - dependencies.update([pre['name']]) - else: - deps = _prepare_deps(pre.get('dependencies', [])) - dependencies.update(deps) - for post in container.get('post', []): - if post.get('type') != 'single': - deps = _prepare_deps(post.get('dependencies', [])) - dependencies.update(deps) - return list(dependencies) - - -def _parse_pre_and_post_deps(service_map): - """Parses service map and finds pres and their dependencies.""" - deps = {} - for container in service_map['service']['containers']: - for pre in container.get('pre', []): - pre_deps = _prepare_deps(pre.get('dependencies', [])) - deps[pre['name']] = Node(pre['name'], - 'job', - pre_deps, - service_map['service']['name']) - - for post in container.get('post', []): - if post.get('type') == 'single': - post_deps = _prepare_deps(post.get('dependencies', [])) - post_deps.append(service_map['service']['name']) - deps[post['name']] = Node(post['name'], - 'job', - post_deps, - service_map['service']['name']) - return deps - - -def _calculate_service_deps(service_name, deps_map): - if service_name not in deps_map: - msg = "Wrong component name '{}'".format(service_name) - LOG.error(msg) - sys.exit(1) - deps = set() - job_parents = set() - current_iteration_set = {deps_map[service_name]} - - while current_iteration_set: - next_iteration_set = set() - for dep in current_iteration_set: - if deps_map[dep.name].is_service(): - deps.update([deps_map[dep.name]]) - else: - job_parents.update([dep.job_parent]) - - for dep in current_iteration_set: - for dependency in deps_map[dep.name].dependencies: - next_iteration_set.update([deps_map[dependency]]) - current_iteration_set = next_iteration_set - - deps = {dep.name for dep in deps} - return deps, job_parents - - -def get_deps(components, components_map=None): - deps_map = get_deps_map(components_map) - result_deps = set() - for service_name in components: - deps, job_parents = _calculate_service_deps(service_name, deps_map) - checked = {service_name} - - while True: - deps.update(job_parents) - if not job_parents - checked: - break - for parent in job_parents - checked: - parent_deps, parent_parents = _calculate_service_deps( - parent, deps_map) - deps.update(parent_deps) - checked.update(job_parents - checked) - job_parents.update(parent_parents) - result_deps.update(deps) - result_deps.add('etcd') - - return result_deps - set(components) + for component in components: + dependencies.update(get_service_deps(deps_graph, component)) + dependencies.add("etcd") + return dependencies - set(components) def show_dep(components): diff --git a/fuel_ccp/deploy.py b/fuel_ccp/deploy.py index 6800f380..127519e3 100644 --- a/fuel_ccp/deploy.py +++ b/fuel_ccp/deploy.py @@ -193,10 +193,10 @@ def _create_job_wfs(container, service_name): wfs = {} for job in container.get("pre", ()): if _is_single_job(job): - wfs.update(_create_job_wf(job, service_name)) + wfs.update(_create_job_wf(job, service_name, container)) for job in container.get("post", ()): if _is_single_job(job): - wfs.update(_create_job_wf(job, service_name, True)) + wfs.update(_create_job_wf(job, service_name, container, True)) return wfs @@ -305,7 +305,8 @@ def _get_job(service, container, job, component_name, topology): cont_spec = templates.serialize_job_container_spec(container, job) pod_spec = templates.serialize_job_pod_spec(service, job, cont_spec, affinity) - job_spec = templates.serialize_job(job["name"], pod_spec, component_name, + job_name = "%s-%s" % (service["name"], job["name"]) + job_spec = templates.serialize_job(job_name, pod_spec, component_name, service["name"]) return job_spec @@ -317,12 +318,12 @@ def _create_command(workflow, cmd): workflow.append(cmd_flow) -def _create_job_wf(job, service_name, post=False): +def _create_job_wf(job, service_name, cont, post=False): wrk = {} wrk["name"] = "%s/%s" % (service_name, job["name"]) wrk["dependencies"] = job.get("dependencies", []) if post: - wrk["dependencies"].append(service_name) + wrk["dependencies"].append("%s/%s" % (service_name, cont["name"])) wrk["job"] = {} _fill_cmd(wrk["job"], job) _push_files_to_workflow(wrk, job.get("files")) @@ -484,7 +485,7 @@ def _create_openrc(config): "export OS_PASSWORD=%s" % config['openstack']['user_password'], "export OS_IDENTITY_API_VERSION=3", "export OS_AUTH_URL=%s/v3" % - utils.address('keystone', config['keystone']['public_port'], True, + utils.address({}, 'keystone', config['keystone']['public_port'], True, True) ] if config['security']['tls']['create_certificates']: @@ -616,17 +617,6 @@ def _create_registry_secret(): kubernetes.process_object(secret) -def process_dependencies(service, deps_map): - containers = service['service_content']['service']['containers'] - for cont in containers: - for cmd in itertools.chain( - cont.get('pre', []), [cont['daemon']], - cont.get('post', [])): - cmd['dependencies'] = ["%s/%s" % ( - deps_map[dep.split(':')[0]], dep) for dep in cmd.get( - 'dependencies', [])] - - def deploy_components(components_map, components): topology = _make_topology(CONF.nodes, CONF.roles, CONF.replicas) @@ -656,14 +646,11 @@ def deploy_components(components_map, components): exports_cm = _create_exports_configmap(exports_map) exports_ctx = {'files_header': j2_imports_files_header, 'map': exports_map} - deps_map = utils.get_dependencies_map(components_map) - configmaps = (start_script_cm, exports_cm) upgrading_components = {} for service_name in components: service = components_map[service_name] - process_dependencies(service, deps_map) service["service_content"]['service']['exports_ctx'] = exports_ctx objects_gen = parse_role(service, topology, configmaps) objects = list(itertools.chain.from_iterable(objects_gen)) diff --git a/fuel_ccp/tests/common/service-rendered-example-custom.yaml b/fuel_ccp/tests/common/service-rendered-example-custom.yaml index bfefbed0..997ad236 100644 --- a/fuel_ccp/tests/common/service-rendered-example-custom.yaml +++ b/fuel_ccp/tests/common/service-rendered-example-custom.yaml @@ -18,8 +18,6 @@ service: - name: chown-logs-dir command: "sudo /bin/chown keystone:keystone /var/log/ccp/keystone" - name: keystone-db-create - dependencies: - - galera type: single command: mysql -u root -pdb_root_password_custom -h galera -e "create database keystone_db_name_custom; @@ -28,14 +26,14 @@ service: files: - keystone-conf dependencies: - - keystone-db-create + - keystone/keystone-db-create type: single command: keystone-manage db_sync - name: keystone-db-bootstrap files: - keystone-conf dependencies: - - keystone-db-sync + - keystone/keystone-db-sync type: single command: keystone-manage bootstrap --bootstrap-password os_user_password_custom @@ -48,8 +46,6 @@ service: --bootstrap-internal-url http://keystone:keystone_public_port_custom daemon: - dependencies: - - memcached files: - keystone-conf - wsgi-keystone-conf diff --git a/fuel_ccp/tests/common/service-rendered-example-default.yaml b/fuel_ccp/tests/common/service-rendered-example-default.yaml index ece627a8..7921db52 100644 --- a/fuel_ccp/tests/common/service-rendered-example-default.yaml +++ b/fuel_ccp/tests/common/service-rendered-example-default.yaml @@ -18,8 +18,6 @@ service: - name: chown-logs-dir command: "sudo /bin/chown keystone:keystone /var/log/ccp/keystone" - name: keystone-db-create - dependencies: - - galera type: single command: mysql -u root -pdb_root_password_default -h galera -e "create database keystone_db_name_default; @@ -28,14 +26,14 @@ service: files: - keystone-conf dependencies: - - keystone-db-create + - keystone/keystone-db-create type: single command: keystone-manage db_sync - name: keystone-db-bootstrap files: - keystone-conf dependencies: - - keystone-db-sync + - keystone/keystone-db-sync type: single command: keystone-manage bootstrap --bootstrap-password os_user_password_default @@ -48,8 +46,6 @@ service: --bootstrap-internal-url http://keystone:keystone_public_port_default daemon: - dependencies: - - memcached files: - keystone-conf - wsgi-keystone-conf diff --git a/fuel_ccp/tests/common/test_repo_dir/component/service/service-example.yaml b/fuel_ccp/tests/common/test_repo_dir/component/service/service-example.yaml index 2b547081..1af42434 100644 --- a/fuel_ccp/tests/common/test_repo_dir/component/service/service-example.yaml +++ b/fuel_ccp/tests/common/test_repo_dir/component/service/service-example.yaml @@ -19,8 +19,6 @@ service: - name: chown-logs-dir command: "sudo /bin/chown {{ service_name }}:{{ service_name }} /var/log/ccp/{{ service_name }}" - name: {{ service_name }}-db-create - dependencies: - - galera type: single command: mysql -u root -p{{ db_root_password }} -h galera -e "create database {{ keystone_db_name }}; @@ -49,8 +47,6 @@ service: --bootstrap-internal-url http://{{ service_name }}:{{ keystone_public_port }} daemon: - dependencies: - - memcached files: - {{ service_name }}-conf - wsgi-{{ service_name }}-conf diff --git a/fuel_ccp/tests/common/test_utils.py b/fuel_ccp/tests/common/test_utils.py index ee646acf..d2dad818 100644 --- a/fuel_ccp/tests/common/test_utils.py +++ b/fuel_ccp/tests/common/test_utils.py @@ -42,7 +42,7 @@ class TestUtils(base.TestCase): res = ( utils.get_deploy_components_info()["keystone"]["service_content"] ) - + print(yaml.dump(res, default_flow_style=False)) with open(os.path.join(base_dir, "service-rendered-example-default.yaml")) as f: expected = yaml.load(f) @@ -171,4 +171,4 @@ class TestAddress(testscenarios.WithScenarios, base.TestCase): self.conf.configs._merge(prepared_conf) self.assertEqual(self.address, utils.address( - 'service', self.port, self.external, self.with_scheme)) + {}, 'service', self.port, self.external, self.with_scheme)) diff --git a/fuel_ccp/tests/test_deploy.py b/fuel_ccp/tests/test_deploy.py index 676c1e42..124f2a67 100644 --- a/fuel_ccp/tests/test_deploy.py +++ b/fuel_ccp/tests/test_deploy.py @@ -373,7 +373,7 @@ class TestDeployParseWorkflow(base.TestCase): "eric-mom": { "workflow": { "name": "south-park/eric-mom", - "dependencies": ["eric-dad", "south-park"], + "dependencies": ["eric-dad", "south-park/kenny"], "files": [ { "name": "eric",