diff --git a/etc/topology-example.yaml b/etc/topology-example.yaml new file mode 100644 index 00000000..27e17f31 --- /dev/null +++ b/etc/topology-example.yaml @@ -0,0 +1,44 @@ +nodes: + node1: + roles: + - controller + - lma + - lma-controller + - openvswitch + node[2-3]: + roles: + - compute + - lma + - openvswitch +roles: + controller: + - etcd + - glance-api + - glance-registry + - horizon + - keystone + - mariadb + - memcached + - neutron-dhcp-agent + - neutron-l3-agent + - neutron-metadata-agent + - neutron-server + - nova-api + - nova-conductor + - nova-consoleauth + - nova-novncproxy + - nova-scheduler + - rabbitmq + compute: + - nova-compute + - nova-libvirt + openvswitch: + - neutron-openvswitch-agent + - openvswitch-db + - openvswitch-vswitchd + lma-controller: + - elasticsearch + - influxdb + - kibana + lma: + - heka diff --git a/fuel_ccp/build.py b/fuel_ccp/build.py index f4688970..5b04322e 100644 --- a/fuel_ccp/build.py +++ b/fuel_ccp/build.py @@ -243,7 +243,7 @@ def _get_config(): if CONF.registry.address: cfg['namespace'] = '%s/%s' % (CONF.registry.address, cfg['namespace']) - cfg.update(utils.get_global_parameters('versions')) + cfg.update(utils.get_global_parameters('versions')["versions"]) return cfg diff --git a/fuel_ccp/common/utils.py b/fuel_ccp/common/utils.py index 96c1ce28..ef1b8961 100644 --- a/fuel_ccp/common/utils.py +++ b/fuel_ccp/common/utils.py @@ -23,7 +23,7 @@ def get_resource_path(path): return pkg_resources.resource_filename(fuel_ccp.version_info.package, path) -def get_global_parameters(config_group): +def get_global_parameters(*config_groups): cfg = {} components = list(CONF.repositories.names) paths = [] @@ -44,7 +44,10 @@ def get_global_parameters(config_group): if os.path.isfile(path): LOG.debug("Adding parameters from \"%s\"", path) with open(path, "r") as f: - cfg.update(yaml.load(f).get(config_group, {})) + data = yaml.load(f) + for group in config_groups: + cfg.setdefault(group, {}) + cfg[group].update(data.get(group, {})) else: LOG.warning("\"%s\" not found, skipping", path) diff --git a/fuel_ccp/deploy.py b/fuel_ccp/deploy.py index 96f1ed70..5c5f1884 100644 --- a/fuel_ccp/deploy.py +++ b/fuel_ccp/deploy.py @@ -35,6 +35,10 @@ def _expand_files(service, files): def parse_role(service_dir, role, config): service = role["service"] + if service["name"] not in config.get("topology", {}): + LOG.info("Service %s not in topology config, skipping deploy", + service["name"]) + return LOG.info("Using service %s", service["name"]) _expand_files(service, role.get("files")) @@ -52,14 +56,17 @@ def parse_role(service_dir, role, config): _create_post_jobs(service, cont) cont_spec = templates.serialize_daemon_pod_spec(service) + affinity = templates.serialize_affinity(service, config["topology"]) if service.get("daemonset", False): - obj = templates.serialize_daemonset(service["name"], cont_spec) + obj = templates.serialize_daemonset(service["name"], cont_spec, + affinity) else: - obj = templates.serialize_deployment(service["name"], cont_spec) + obj = templates.serialize_deployment(service["name"], cont_spec, + affinity) kubernetes.create_object_from_definition(obj) - _create_service(service, config) + _create_service(service, config["configs"]) def _parse_workflows(service): @@ -264,6 +271,44 @@ def deploy_component(component, config): parse_role(service_dir, role_obj, config) +def _make_topology(nodes, roles): + failed = False + # TODO(sreshetniak): move it to validation + if not nodes: + LOG.error("Nodes section is not specified in configs") + failed = True + if not roles: + LOG.error("Roles section is not specified in configs") + failed = True + if failed: + raise RuntimeError("Failed to create topology for services") + + # TODO(sreshetniak): add validation + k8s_nodes = kubernetes.list_k8s_nodes() + + def find_match(glob): + matcher = re.compile(glob) + nodes = [] + for node in k8s_nodes: + match = matcher.match(node) + if match: + nodes.append(match.group(0)) + return nodes + + roles_to_node = {} + for node in nodes.keys(): + matched_nodes = find_match(node) + for role in nodes[node]["roles"]: + roles_to_node.setdefault(role, []) + roles_to_node[role].extend(matched_nodes) + service_to_node = {} + for role in roles.keys(): + for svc in roles[role]: + service_to_node.setdefault(svc, []) + service_to_node[svc].extend(roles_to_node[role]) + return service_to_node + + def _create_namespace(namespace): if CONF.action.dry_run: return @@ -284,12 +329,15 @@ def _create_namespace(namespace): def deploy_components(components=None): if components is None: components = CONF.repositories.names - namespace = CONF.kubernetes.namespace + config = utils.get_global_parameters("configs", "nodes", "roles") + config["topology"] = _make_topology(config.get("nodes"), + config.get("roles")) + + namespace = CONF.kubernetes.namespace _create_namespace(namespace) - config = utils.get_global_parameters('configs') - _create_globals_configmap(config) + _create_globals_configmap(config["configs"]) _create_start_script_configmap() for component in components: diff --git a/fuel_ccp/kubernetes.py b/fuel_ccp/kubernetes.py index 81174276..1db3581d 100644 --- a/fuel_ccp/kubernetes.py +++ b/fuel_ccp/kubernetes.py @@ -73,6 +73,15 @@ def get_v1_api(client): return apiv_api.ApivApi(client) +def list_k8s_nodes(): + api = get_v1_api(get_client()) + resp = api.list_namespaced_node() + nodes = [] + for node in resp.items: + nodes.append(node.metadata.name) + return nodes + + def handle_exists(fct, *args, **kwargs): try: fct(*args, **kwargs) diff --git a/fuel_ccp/templates.py b/fuel_ccp/templates.py index ef8f5b2d..8565d674 100644 --- a/fuel_ccp/templates.py +++ b/fuel_ccp/templates.py @@ -1,4 +1,4 @@ -import copy +import json from oslo_config import cfg @@ -133,8 +133,6 @@ def serialize_daemon_pod_spec(service): if service.get("host-net"): cont_spec["hostNetwork"] = True - if service.get("node-selector"): - cont_spec["nodeSelector"] = copy.deepcopy(service["node-selector"]) return cont_spec @@ -240,7 +238,7 @@ def serialize_job(name, spec): } -def serialize_deployment(name, spec): +def serialize_deployment(name, spec, affinity): return { "apiVersion": "extensions/v1beta1", "kind": "Deployment", @@ -251,6 +249,7 @@ def serialize_deployment(name, spec): "replicas": 1, "template": { "metadata": { + "annotations": affinity, "labels": { "mcp": "true", "app": name @@ -262,7 +261,7 @@ def serialize_deployment(name, spec): } -def serialize_daemonset(name, spec): +def serialize_daemonset(name, spec, affinity): return { "apiVersion": "extensions/v1beta1", "kind": "DaemonSet", @@ -272,6 +271,7 @@ def serialize_daemonset(name, spec): "spec": { "template": { "metadata": { + "annotations": affinity, "labels": { "mcp": "true", "app": name @@ -283,6 +283,23 @@ def serialize_daemonset(name, spec): } +def serialize_affinity(service, topology): + policy = { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{ + "matchExpressions": [{ + "key": "kubernetes.io/hostname", + "operator": "In", + "values": topology[service["name"]] + }] + }] + } + } + } + return {"scheduler.alpha.kubernetes.io/affinity": json.dumps(policy)} + + def serialize_service(name, ports): ports_spec = [] for port in ports: diff --git a/fuel_ccp/tests/test_deploy.py b/fuel_ccp/tests/test_deploy.py index e05ca2d6..683ac2a5 100644 --- a/fuel_ccp/tests/test_deploy.py +++ b/fuel_ccp/tests/test_deploy.py @@ -266,3 +266,45 @@ class TestDeployParseWorkflow(base.TestCase): } } self.assertDictEqual(expected_workflows, workflow) + + +class TestDeployMakeTopology(base.TestCase): + def test_make_empty_topology(self): + self.assertRaises(RuntimeError, + deploy._make_topology, None, None) + self.assertRaises(RuntimeError, + deploy._make_topology, None, {"spam": "eggs"}) + self.assertRaises(RuntimeError, + deploy._make_topology, {"spam": "eggs"}, None) + + def test_make_topology(self): + nodes = { + "node1": { + "roles": ["controller"] + }, + "node[2-3]": { + "roles": ["compute"] + } + } + roles = { + "controller": [ + "mysql", + "keystone" + ], + "compute": [ + "nova-compute", + "libvirtd" + ] + } + + node_list = ["node1", "node2", "node3"] + expected_topology = { + "mysql": ["node1"], + "keystone": ["node1"], + "nova-compute": ["node2", "node3"], + "libvirtd": ["node2", "node3"] + } + with mock.patch("fuel_ccp.kubernetes.list_k8s_nodes") as p: + p.return_value = node_list + topology = deploy._make_topology(nodes, roles) + self.assertDictEqual(expected_topology, topology)