diff --git a/nailgun/nailgun/objects/cluster.py b/nailgun/nailgun/objects/cluster.py index 5f39d92712..d3626fbb69 100644 --- a/nailgun/nailgun/objects/cluster.py +++ b/nailgun/nailgun/objects/cluster.py @@ -40,7 +40,7 @@ from nailgun.objects.plugin import ClusterPlugins from nailgun.objects import Release from nailgun.objects.serializers.cluster import ClusterSerializer from nailgun.plugins.manager import PluginManager -from nailgun.plugins.merge_policies import NetworkRoleMergePolicy +from nailgun.policy.merge import NetworkRoleMergePolicy from nailgun.settings import settings from nailgun.utils import AttributesGenerator from nailgun.utils import dict_merge diff --git a/nailgun/nailgun/orchestrator/deployment_graph.py b/nailgun/nailgun/orchestrator/deployment_graph.py index fcc64e8c8d..9b0e459012 100644 --- a/nailgun/nailgun/orchestrator/deployment_graph.py +++ b/nailgun/nailgun/orchestrator/deployment_graph.py @@ -29,6 +29,7 @@ from nailgun.logger import logger from nailgun import objects from nailgun.orchestrator import priority_serializers as ps from nailgun.orchestrator.tasks_serializer import TaskSerializers +from nailgun.policy.name_match import NameMatchingPolicy class DeploymentGraph(nx.DiGraph): @@ -88,6 +89,8 @@ class DeploymentGraph(nx.DiGraph): for task in tasks: self.add_task(task) + self._update_dependencies() + def add_task(self, task): self.add_node(task['id'], **task) @@ -98,17 +101,37 @@ class DeploymentGraph(nx.DiGraph): for req in task.get('requires', ()): self.add_edge(req, task['id']) - # tasks and groups should be used for declaring dependencies between - # tasks and roles (which are simply group of tasks) - for req in task.get('groups', ()): - self.add_edge(task['id'], req) - for req in task.get('tasks', ()): - self.add_edge(req, task['id']) - # FIXME(dshulyak) remove it after change in library will be merged if 'stage' in task: self.add_edge(task['id'], task['stage']) + def _update_dependencies(self): + """Create dependencies that rely on regexp matching.""" + + for task in six.itervalues(self.node): + # tasks and groups should be used for declaring dependencies + # between tasks and roles (which are simply group of tasks) + available_groups = self.get_groups_subgraph().nodes() + for group in task.get('groups', ()): + pattern = NameMatchingPolicy.create(group) + not_matched = [] + for available_group in available_groups: + if pattern.match(available_group): + self.add_edge(task['id'], available_group) + else: + not_matched.append(available_group) + # Add dependency for non-existing group which will be + # resolved in DeploymentGraphValidator + if len(available_groups) == len(not_matched): + self.add_edge(task['id'], group) + logger.warning( + 'Group "%s" is an invalid dependency', group) + + available_groups = not_matched + + for req in task.get('tasks', ()): + self.add_edge(req, task['id']) + def is_acyclic(self): """Verify that graph doesnot contain any cycles in it.""" return nx.is_directed_acyclic_graph(self) @@ -134,8 +157,9 @@ class DeploymentGraph(nx.DiGraph): return result def get_groups_subgraph(self): - roles = [t['id'] for t in self.node.values() - if t['type'] == consts.ORCHESTRATOR_TASK_TYPES.group] + """Return subgraph containing all the groups of tasks.""" + roles = [t['id'] for t in six.itervalues(self.node) + if t.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group] return self.subgraph(roles) def get_group_tasks(self, group_name): diff --git a/nailgun/nailgun/orchestrator/task_based_deploy.py b/nailgun/nailgun/orchestrator/task_based_deploy.py index b743db970d..743c86134a 100644 --- a/nailgun/nailgun/orchestrator/task_based_deploy.py +++ b/nailgun/nailgun/orchestrator/task_based_deploy.py @@ -28,7 +28,7 @@ from nailgun.orchestrator.tasks_serializer import CreateVMsOnCompute from nailgun.orchestrator.tasks_serializer import StandartConfigRolesHook from nailgun.orchestrator.tasks_serializer import TaskSerializers from nailgun.orchestrator.tasks_templates import make_noop_task -from nailgun.utils.role_resolver import NameMatchPolicy +from nailgun.utils.role_resolver import NameMatchingPolicy from nailgun.utils.role_resolver import NullResolver from nailgun.utils.role_resolver import RoleResolver @@ -470,7 +470,7 @@ class TasksSerializer(object): :param is_required_for: means task from required_for section """ found = False - match_policy = NameMatchPolicy.create(name) + match_policy = NameMatchingPolicy.create(name) for node_id in node_ids: applied_tasks = set() for task_name in self.tasks_per_node[node_id]: diff --git a/nailgun/nailgun/policy/__init__.py b/nailgun/nailgun/policy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nailgun/nailgun/plugins/merge_policies.py b/nailgun/nailgun/policy/merge.py similarity index 99% rename from nailgun/nailgun/plugins/merge_policies.py rename to nailgun/nailgun/policy/merge.py index f41240fb07..64ebef16e9 100644 --- a/nailgun/nailgun/plugins/merge_policies.py +++ b/nailgun/nailgun/policy/merge.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright 2015 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/nailgun/nailgun/policy/name_match.py b/nailgun/nailgun/policy/name_match.py new file mode 100644 index 0000000000..4726e94fd4 --- /dev/null +++ b/nailgun/nailgun/policy/name_match.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import re +import six + + +@six.add_metaclass(abc.ABCMeta) +class NameMatchingPolicy(object): + @abc.abstractmethod + def match(self, name): + """Tests that name is acceptable. + + :param name: the name to test + :type name: str + :returns: True if yes otherwise False + """ + + @staticmethod + def create(pattern): + """Makes name matching policy. + + the string wrapped with '/' treats as pattern + '/abc/' - pattern + 'abc' - the string for exact match + + :param pattern: the pattern to match + :return: the NameMatchPolicy instance + """ + if pattern.startswith("/") and pattern.endswith("/"): + return PatternMatchingPolicy(pattern[1:-1]) + return ExactMatchingPolicy(pattern) + + +class ExactMatchingPolicy(NameMatchingPolicy): + """Tests that name exact match to argument.""" + + def __init__(self, name): + """Initializes. + + :param name: the name to match + """ + self.name = name + + def match(self, name): + return self.name == name + + +class PatternMatchingPolicy(NameMatchingPolicy): + """Tests that pattern matches to argument.""" + + def __init__(self, pattern): + self.pattern = re.compile(pattern) + + def match(self, name): + return self.pattern.match(name) diff --git a/nailgun/nailgun/test/unit/test_graph_serializer.py b/nailgun/nailgun/test/unit/test_graph_serializer.py index b39a94751f..dc9344f056 100644 --- a/nailgun/nailgun/test/unit/test_graph_serializer.py +++ b/nailgun/nailgun/test/unit/test_graph_serializer.py @@ -15,6 +15,7 @@ # under the License. from collections import defaultdict +import copy from itertools import groupby import mock @@ -105,7 +106,6 @@ SUBTASKS = """ puppet_manifest: run_setup_network.pp puppet_modules: /etc/puppet timeout: 120 - - id: setup_anything requires: [pre_deployment_start] required_for: [pre_deployment] @@ -116,6 +116,18 @@ SUBTASKS = """ requires: [setup_anything] """ +SUBTASKS_WITH_REGEXP = """ +- id: setup_something + type: puppet + groups: ['/cinder|compute/', '/(?=controller)(?=^((?!^primary).)*$)/'] + required_for: [deploy_end] + requires: [deploy_start] + parameters: + puppet_manifest: run_setup_something.pp + puppet_modules: /etc/puppet + timeout: 120 +""" + class TestGraphDependencies(base.BaseTestCase): @@ -123,6 +135,7 @@ class TestGraphDependencies(base.BaseTestCase): super(TestGraphDependencies, self).setUp() self.tasks = yaml.load(TASKS) self.subtasks = yaml.load(SUBTASKS) + self.subtasks_with_regexp = yaml.load(SUBTASKS_WITH_REGEXP) self.graph = deployment_graph.DeploymentGraph() def test_build_deployment_graph(self): @@ -149,6 +162,40 @@ class TestGraphDependencies(base.BaseTestCase): ['setup_network', 'install_controller']) +class TestUpdateGraphDependencies(base.BaseTestCase): + + def setUp(self): + super(TestUpdateGraphDependencies, self).setUp() + self.tasks = yaml.load(TASKS) + self.subtasks = yaml.load(SUBTASKS_WITH_REGEXP) + + def test_groups_regexp_resolution(self): + graph = deployment_graph.DeploymentGraph() + graph.add_tasks(self.tasks + self.subtasks) + self.assertItemsEqual( + graph.succ['setup_something'], + {'deploy_end': {}, 'cinder': {}, 'compute': {}, 'controller': {}}) + + def test_support_for_all_groups(self): + graph = deployment_graph.DeploymentGraph() + subtasks = copy.deepcopy(self.subtasks) + subtasks[0]['groups'] = ['/.*/'] + graph.add_tasks(self.tasks + subtasks) + self.assertItemsEqual( + graph.succ['setup_something'], + {'deploy_end': {}, 'primary-controller': {}, 'network': {}, + 'cinder': {}, 'compute': {}, 'controller': {}}) + + def test_simple_string_in_group(self): + graph = deployment_graph.DeploymentGraph() + subtasks = copy.deepcopy(self.subtasks) + subtasks[0]['groups'] = ['controller'] + graph.add_tasks(self.tasks + subtasks) + self.assertItemsEqual( + graph.succ['setup_something'], + {'deploy_end': {}, 'controller': {}}) + + class TestAddDependenciesToNodes(base.BaseTestCase): def setUp(self): diff --git a/nailgun/nailgun/test/unit/test_plugins_merge_policies.py b/nailgun/nailgun/test/unit/test_policies.py similarity index 76% rename from nailgun/nailgun/test/unit/test_plugins_merge_policies.py rename to nailgun/nailgun/test/unit/test_policies.py index 0c37dbe60d..f8ef41866e 100644 --- a/nailgun/nailgun/test/unit/test_plugins_merge_policies.py +++ b/nailgun/nailgun/test/unit/test_policies.py @@ -17,11 +17,14 @@ from nailgun import consts from nailgun.errors import errors -from nailgun.plugins.merge_policies import NetworkRoleMergePolicy -from nailgun.test.base import BaseTestCase +from nailgun.policy.merge import NetworkRoleMergePolicy +from nailgun.policy.name_match import ExactMatchingPolicy +from nailgun.policy.name_match import NameMatchingPolicy +from nailgun.policy.name_match import PatternMatchingPolicy +from nailgun.test.base import BaseUnitTest -class TestNetworkRoleMergePolicy(BaseTestCase): +class TestNetworkRoleMergePolicy(BaseUnitTest): def setUp(self): super(TestNetworkRoleMergePolicy, self).setUp() self.policy = NetworkRoleMergePolicy() @@ -88,3 +91,17 @@ class TestNetworkRoleMergePolicy(BaseTestCase): vip=[{"name": "test", "value": 2}] ) ) + + +class TestNameMatchingPolicy(BaseUnitTest): + def test_exact_match(self): + match_policy = NameMatchingPolicy.create("controller") + self.assertIsInstance(match_policy, ExactMatchingPolicy) + self.assertTrue(match_policy.match("controller")) + self.assertFalse(match_policy.match("controller1")) + + def test_pattern_match(self): + match_policy = NameMatchingPolicy.create("/controller/") + self.assertIsInstance(match_policy, PatternMatchingPolicy) + self.assertTrue(match_policy.match("controller")) + self.assertTrue(match_policy.match("controller1")) diff --git a/nailgun/nailgun/test/unit/test_role_resolver.py b/nailgun/nailgun/test/unit/test_role_resolver.py index 35385a2602..32b8a02090 100644 --- a/nailgun/nailgun/test/unit/test_role_resolver.py +++ b/nailgun/nailgun/test/unit/test_role_resolver.py @@ -22,20 +22,6 @@ from nailgun.test.base import BaseUnitTest from nailgun.utils import role_resolver -class TestNameMatchPolicy(BaseUnitTest): - def test_exact_match(self): - match_policy = role_resolver.NameMatchPolicy.create("controller") - self.assertIsInstance(match_policy, role_resolver.ExactMatch) - self.assertTrue(match_policy.match("controller")) - self.assertFalse(match_policy.match("controller1")) - - def test_pattern_match(self): - match_policy = role_resolver.NameMatchPolicy.create("/controller/") - self.assertIsInstance(match_policy, role_resolver.PatternMatch) - self.assertTrue(match_policy.match("controller")) - self.assertTrue(match_policy.match("controller1")) - - class TestPatternBasedRoleResolver(BaseUnitTest): @classmethod def setUpClass(cls): diff --git a/nailgun/nailgun/utils/role_resolver.py b/nailgun/nailgun/utils/role_resolver.py index 4e1e815c8b..8aae13deff 100644 --- a/nailgun/nailgun/utils/role_resolver.py +++ b/nailgun/nailgun/utils/role_resolver.py @@ -16,64 +16,13 @@ import abc from collections import defaultdict -import re import six from nailgun import consts from nailgun.logger import logger from nailgun import objects - - -@six.add_metaclass(abc.ABCMeta) -class NameMatchPolicy(object): - @abc.abstractmethod - def match(self, name): - """Tests that name is acceptable. - - :param name: the name to test - :type name: str - :returns: True if yes otherwise False - """ - - @staticmethod - def create(pattern): - """Makes name match policy. - - the string wrapped with '/' treats as pattern - '/abc/' - pattern - 'abc' - the string for exact match - - :param pattern: the pattern to match - :return: the NameMatchPolicy instance - """ - if pattern.startswith("/") and pattern.endswith("/"): - return PatternMatch(pattern[1:-1]) - return ExactMatch(pattern) - - -class ExactMatch(NameMatchPolicy): - """Tests that name exact match to argument.""" - - def __init__(self, name): - """Initializes. - - :param name: the name to match - """ - self.name = name - - def match(self, name): - return self.name == name - - -class PatternMatch(NameMatchPolicy): - """Tests that pattern matches to argument.""" - - def __init__(self, patten): - self.pattern = re.compile(patten) - - def match(self, name): - return self.pattern.match(name) +from nailgun.policy.name_match import NameMatchingPolicy @six.add_metaclass(abc.ABCMeta) @@ -140,7 +89,7 @@ class RoleResolver(BaseRoleResolver): elif isinstance(roles, (list, tuple)): result = set() for role in roles: - pattern = NameMatchPolicy.create(role) + pattern = NameMatchingPolicy.create(role) for node_role, nodes_ids in six.iteritems(self.__mapping): if pattern.match(node_role): result.update(nodes_ids)