openstack
/
fuel-web
Code Issues Proposed changes

Added support of regexp patterns for groups in deployment_tasks.yaml 

Regular expressions (perl format) should be used to specify wildcard.
'/' used on the end of group name to distinguish regexp from simple string.

groups: '/.*/' # any group
groups: '/^(?!controller$)/' # all groups except controller
groups: '/(?=controller)(?=^((?!^primary).)*$)/' # all controller,
        except primary-controller
groups: '/mongo|compute/' # mongo or compute

DocImpact
Closes-Bug: #1480930

Change-Id: I1f279fbd585775fc7b5af1ecbbaa24be21d831c9
This commit is contained in:
Alexander Rozdymakha 2015-11-05 18:01:01 +02:00 committed by Andriy Popovych
parent 3525483860
commit e434546d68
10 changed files with 178 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nailgun/nailgun/objects/cluster.py
View File

@ -40,7 +40,7 @@ from nailgun.objects.plugin import ClusterPlugins
from nailgun.objects import Release
from nailgun.objects.serializers.cluster import ClusterSerializer
from nailgun.plugins.manager import PluginManager
from nailgun.plugins.merge_policies import NetworkRoleMergePolicy
from nailgun.policy.merge import NetworkRoleMergePolicy
from nailgun.settings import settings
from nailgun.utils import AttributesGenerator
from nailgun.utils import dict_merge

42
nailgun/nailgun/orchestrator/deployment_graph.py
View File

@ -29,6 +29,7 @@ from nailgun.logger import logger
from nailgun import objects
from nailgun.orchestrator import priority_serializers as ps
from nailgun.orchestrator.tasks_serializer import TaskSerializers
from nailgun.policy.name_match import NameMatchingPolicy
class DeploymentGraph(nx.DiGraph):
@ -88,6 +89,8 @@ class DeploymentGraph(nx.DiGraph):
for task in tasks:
self.add_task(task)
self._update_dependencies()
def add_task(self, task):
self.add_node(task['id'], **task)
@ -98,17 +101,37 @@ class DeploymentGraph(nx.DiGraph):
for req in task.get('requires', ()):
self.add_edge(req, task['id'])
# tasks and groups should be used for declaring dependencies between
# tasks and roles (which are simply group of tasks)
for req in task.get('groups', ()):
self.add_edge(task['id'], req)
for req in task.get('tasks', ()):
self.add_edge(req, task['id'])
# FIXME(dshulyak) remove it after change in library will be merged
if 'stage' in task:
self.add_edge(task['id'], task['stage'])
def _update_dependencies(self):
"""Create dependencies that rely on regexp matching."""
for task in six.itervalues(self.node):
# tasks and groups should be used for declaring dependencies
# between tasks and roles (which are simply group of tasks)
available_groups = self.get_groups_subgraph().nodes()
for group in task.get('groups', ()):
pattern = NameMatchingPolicy.create(group)
not_matched = []
for available_group in available_groups:
if pattern.match(available_group):
self.add_edge(task['id'], available_group)
else:
not_matched.append(available_group)
# Add dependency for non-existing group which will be
# resolved in DeploymentGraphValidator
if len(available_groups) == len(not_matched):
self.add_edge(task['id'], group)
logger.warning(
'Group "%s" is an invalid dependency', group)
available_groups = not_matched
for req in task.get('tasks', ()):
self.add_edge(req, task['id'])
def is_acyclic(self):
"""Verify that graph doesnot contain any cycles in it."""
return nx.is_directed_acyclic_graph(self)
@ -134,8 +157,9 @@ class DeploymentGraph(nx.DiGraph):
return result
def get_groups_subgraph(self):
roles = [t['id'] for t in self.node.values()
if t['type'] == consts.ORCHESTRATOR_TASK_TYPES.group]
"""Return subgraph containing all the groups of tasks."""
roles = [t['id'] for t in six.itervalues(self.node)
if t.get('type') == consts.ORCHESTRATOR_TASK_TYPES.group]
return self.subgraph(roles)
def get_group_tasks(self, group_name):

4
nailgun/nailgun/orchestrator/task_based_deploy.py
View File

@ -28,7 +28,7 @@ from nailgun.orchestrator.tasks_serializer import CreateVMsOnCompute
from nailgun.orchestrator.tasks_serializer import StandartConfigRolesHook
from nailgun.orchestrator.tasks_serializer import TaskSerializers
from nailgun.orchestrator.tasks_templates import make_noop_task
from nailgun.utils.role_resolver import NameMatchPolicy
from nailgun.utils.role_resolver import NameMatchingPolicy
from nailgun.utils.role_resolver import NullResolver
from nailgun.utils.role_resolver import RoleResolver
@ -470,7 +470,7 @@ class TasksSerializer(object):
:param is_required_for: means task from required_for section
"""
found = False
match_policy = NameMatchPolicy.create(name)
match_policy = NameMatchingPolicy.create(name)
for node_id in node_ids:
applied_tasks = set()
for task_name in self.tasks_per_node[node_id]:

0
nailgun/nailgun/policy/__init__.py Normal file
View File

2
nailgun/nailgun/plugins/merge_policies.py → nailgun/nailgun/policy/merge.py
View File

@ -1,3 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

70
nailgun/nailgun/policy/name_match.py Normal file
View File

@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import re
import six
@six.add_metaclass(abc.ABCMeta)
class NameMatchingPolicy(object):
@abc.abstractmethod
def match(self, name):
"""Tests that name is acceptable.
:param name: the name to test
:type name: str
:returns: True if yes otherwise False
"""
@staticmethod
def create(pattern):
"""Makes name matching policy.
the string wrapped with '/' treats as pattern
'/abc/' - pattern
'abc' - the string for exact match
:param pattern: the pattern to match
:return: the NameMatchPolicy instance
"""
if pattern.startswith("/") and pattern.endswith("/"):
return PatternMatchingPolicy(pattern[1:-1])
return ExactMatchingPolicy(pattern)
class ExactMatchingPolicy(NameMatchingPolicy):
"""Tests that name exact match to argument."""
def __init__(self, name):
"""Initializes.
:param name: the name to match
"""
self.name = name
def match(self, name):
return self.name == name
class PatternMatchingPolicy(NameMatchingPolicy):
"""Tests that pattern matches to argument."""
def __init__(self, pattern):
self.pattern = re.compile(pattern)
def match(self, name):
return self.pattern.match(name)

49
nailgun/nailgun/test/unit/test_graph_serializer.py
View File

@ -15,6 +15,7 @@
# under the License.
from collections import defaultdict
import copy
from itertools import groupby
import mock
@ -105,7 +106,6 @@ SUBTASKS = """
puppet_manifest: run_setup_network.pp
puppet_modules: /etc/puppet
timeout: 120
- id: setup_anything
requires: [pre_deployment_start]
required_for: [pre_deployment]
@ -116,6 +116,18 @@ SUBTASKS = """
requires: [setup_anything]
"""
SUBTASKS_WITH_REGEXP = """
- id: setup_something
type: puppet
groups: ['/cinder|compute/', '/(?=controller)(?=^((?!^primary).)*$)/']
required_for: [deploy_end]
requires: [deploy_start]
parameters:
puppet_manifest: run_setup_something.pp
puppet_modules: /etc/puppet
timeout: 120
"""
class TestGraphDependencies(base.BaseTestCase):
@ -123,6 +135,7 @@ class TestGraphDependencies(base.BaseTestCase):
super(TestGraphDependencies, self).setUp()
self.tasks = yaml.load(TASKS)
self.subtasks = yaml.load(SUBTASKS)
self.subtasks_with_regexp = yaml.load(SUBTASKS_WITH_REGEXP)
self.graph = deployment_graph.DeploymentGraph()
def test_build_deployment_graph(self):
@ -149,6 +162,40 @@ class TestGraphDependencies(base.BaseTestCase):
['setup_network', 'install_controller'])
class TestUpdateGraphDependencies(base.BaseTestCase):
def setUp(self):
super(TestUpdateGraphDependencies, self).setUp()
self.tasks = yaml.load(TASKS)
self.subtasks = yaml.load(SUBTASKS_WITH_REGEXP)
def test_groups_regexp_resolution(self):
graph = deployment_graph.DeploymentGraph()
graph.add_tasks(self.tasks + self.subtasks)
self.assertItemsEqual(
graph.succ['setup_something'],
{'deploy_end': {}, 'cinder': {}, 'compute': {}, 'controller': {}})
def test_support_for_all_groups(self):
graph = deployment_graph.DeploymentGraph()
subtasks = copy.deepcopy(self.subtasks)
subtasks[0]['groups'] = ['/.*/']
graph.add_tasks(self.tasks + subtasks)
self.assertItemsEqual(
graph.succ['setup_something'],
{'deploy_end': {}, 'primary-controller': {}, 'network': {},
'cinder': {}, 'compute': {}, 'controller': {}})
def test_simple_string_in_group(self):
graph = deployment_graph.DeploymentGraph()
subtasks = copy.deepcopy(self.subtasks)
subtasks[0]['groups'] = ['controller']
graph.add_tasks(self.tasks + subtasks)
self.assertItemsEqual(
graph.succ['setup_something'],
{'deploy_end': {}, 'controller': {}})
class TestAddDependenciesToNodes(base.BaseTestCase):
def setUp(self):

23
nailgun/nailgun/test/unit/test_plugins_merge_policies.py → nailgun/nailgun/test/unit/test_policies.py
View File

@ -17,11 +17,14 @@
from nailgun import consts
from nailgun.errors import errors
from nailgun.plugins.merge_policies import NetworkRoleMergePolicy
from nailgun.test.base import BaseTestCase
from nailgun.policy.merge import NetworkRoleMergePolicy
from nailgun.policy.name_match import ExactMatchingPolicy
from nailgun.policy.name_match import NameMatchingPolicy
from nailgun.policy.name_match import PatternMatchingPolicy
from nailgun.test.base import BaseUnitTest
class TestNetworkRoleMergePolicy(BaseTestCase):
class TestNetworkRoleMergePolicy(BaseUnitTest):
def setUp(self):
super(TestNetworkRoleMergePolicy, self).setUp()
self.policy = NetworkRoleMergePolicy()
@ -88,3 +91,17 @@ class TestNetworkRoleMergePolicy(BaseTestCase):
vip=[{"name": "test", "value": 2}]
)
)
class TestNameMatchingPolicy(BaseUnitTest):
def test_exact_match(self):
match_policy = NameMatchingPolicy.create("controller")
self.assertIsInstance(match_policy, ExactMatchingPolicy)
self.assertTrue(match_policy.match("controller"))
self.assertFalse(match_policy.match("controller1"))
def test_pattern_match(self):
match_policy = NameMatchingPolicy.create("/controller/")
self.assertIsInstance(match_policy, PatternMatchingPolicy)
self.assertTrue(match_policy.match("controller"))
self.assertTrue(match_policy.match("controller1"))

14
nailgun/nailgun/test/unit/test_role_resolver.py
View File

@ -22,20 +22,6 @@ from nailgun.test.base import BaseUnitTest
from nailgun.utils import role_resolver
class TestNameMatchPolicy(BaseUnitTest):
def test_exact_match(self):
match_policy = role_resolver.NameMatchPolicy.create("controller")
self.assertIsInstance(match_policy, role_resolver.ExactMatch)
self.assertTrue(match_policy.match("controller"))
self.assertFalse(match_policy.match("controller1"))
def test_pattern_match(self):
match_policy = role_resolver.NameMatchPolicy.create("/controller/")
self.assertIsInstance(match_policy, role_resolver.PatternMatch)
self.assertTrue(match_policy.match("controller"))
self.assertTrue(match_policy.match("controller1"))
class TestPatternBasedRoleResolver(BaseUnitTest):
@classmethod
def setUpClass(cls):

55
nailgun/nailgun/utils/role_resolver.py
View File

@ -16,64 +16,13 @@
import abc
from collections import defaultdict
import re
import six
from nailgun import consts
from nailgun.logger import logger
from nailgun import objects
@six.add_metaclass(abc.ABCMeta)
class NameMatchPolicy(object):
@abc.abstractmethod
def match(self, name):
"""Tests that name is acceptable.
:param name: the name to test
:type name: str
:returns: True if yes otherwise False
"""
@staticmethod
def create(pattern):
"""Makes name match policy.
the string wrapped with '/' treats as pattern
'/abc/' - pattern
'abc' - the string for exact match
:param pattern: the pattern to match
:return: the NameMatchPolicy instance
"""
if pattern.startswith("/") and pattern.endswith("/"):
return PatternMatch(pattern[1:-1])
return ExactMatch(pattern)
class ExactMatch(NameMatchPolicy):
"""Tests that name exact match to argument."""
def __init__(self, name):
"""Initializes.
:param name: the name to match
"""
self.name = name
def match(self, name):
return self.name == name
class PatternMatch(NameMatchPolicy):
"""Tests that pattern matches to argument."""
def __init__(self, patten):
self.pattern = re.compile(patten)
def match(self, name):
return self.pattern.match(name)
from nailgun.policy.name_match import NameMatchingPolicy
@six.add_metaclass(abc.ABCMeta)
@ -140,7 +89,7 @@ class RoleResolver(BaseRoleResolver):
elif isinstance(roles, (list, tuple)):
result = set()
for role in roles:
pattern = NameMatchPolicy.create(role)
pattern = NameMatchingPolicy.create(role)
for node_role, nodes_ids in six.iteritems(self.__mapping):
if pattern.match(node_role):
result.update(nodes_ids)