summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2016-08-24 16:08:42 +0000
committerGerrit Code Review <review@openstack.org>2016-08-24 16:08:42 +0000
commit96ca0fdf0db24a84c510029e035cc60460d61a5c (patch)
treeb635b86f3bd8bad4b12f1cd2ddfb6fda6bc88ffb
parent7cf3fe9b3aa0965514e4370e908909e548ff918e (diff)
parent163ce243fbade3dac05eb535ad2987687a57f87d (diff)
Merge "Add pluggable transformations for data migration"
-rw-r--r--cluster_upgrade/tests/test_transformations.py179
-rw-r--r--cluster_upgrade/transformations/__init__.py94
2 files changed, 273 insertions, 0 deletions
diff --git a/cluster_upgrade/tests/test_transformations.py b/cluster_upgrade/tests/test_transformations.py
new file mode 100644
index 0000000..46e0c77
--- /dev/null
+++ b/cluster_upgrade/tests/test_transformations.py
@@ -0,0 +1,179 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13from distutils import version
14
15import mock
16from nailgun.test import base as nailgun_test_base
17import six
18
19from .. import transformations
20
21
22class TestTransformations(nailgun_test_base.BaseUnitTest):
23 def test_get_config(self):
24 config = object()
25
26 class Manager(transformations.Manager):
27 default_config = config
28
29 self.assertIs(config, Manager.get_config('testname'))
30
31 def setup_extension_manager(self, extensions):
32 p = mock.patch("stevedore.ExtensionManager", spec=['__call__'])
33 mock_extman = p.start()
34 self.addCleanup(p.stop)
35
36 def extman(namespace, *args, **kwargs):
37 instance = mock.MagicMock(name=namespace)
38 ext_results = {}
39 for ver, exts in six.iteritems(extensions):
40 if namespace.endswith(ver):
41 ext_results = {name: mock.Mock(name=name, plugin=ext)
42 for name, ext in six.iteritems(exts)}
43 break
44 else:
45 self.fail("Called with unexpected version in namespace: {}, "
46 "expected versions: {}".format(
47 namespace, list(extensions)))
48 instance.__getitem__.side_effect = ext_results.__getitem__
49 return instance
50
51 mock_extman.side_effect = extman
52 return mock_extman
53
54 def test_load_transformers(self):
55 config = {'9.0': ['a', 'b']}
56 extensions = {'9.0': {
57 'a': mock.Mock(name='a'),
58 'b': mock.Mock(name='b'),
59 }}
60 mock_extman = self.setup_extension_manager(extensions)
61
62 res = transformations.Manager.load_transformers('testname', config)
63
64 self.assertEqual(res, [(version.StrictVersion('9.0'), [
65 extensions['9.0']['a'],
66 extensions['9.0']['b'],
67 ])])
68 callback = transformations.reraise_endpoint_load_failure
69 self.assertEqual(mock_extman.mock_calls, [
70 mock.call(
71 'nailgun.cluster_upgrade.transformations.testname.9.0',
72 on_load_failure_callback=callback,
73 ),
74 ])
75
76 def test_load_transformers_empty(self):
77 config = {}
78 extensions = {'9.0': {
79 'a': mock.Mock(name='a'),
80 'b': mock.Mock(name='b'),
81 }}
82 mock_extman = self.setup_extension_manager(extensions)
83
84 res = transformations.Manager.load_transformers('testname', config)
85
86 self.assertEqual(res, [])
87 self.assertEqual(mock_extman.mock_calls, [])
88
89 def test_load_transformers_sorted(self):
90 config = {'9.0': ['a', 'b'], '8.0': ['c']}
91 extensions = {
92 '9.0': {
93 'a': mock.Mock(name='a'),
94 'b': mock.Mock(name='b'),
95 },
96 '8.0': {
97 'c': mock.Mock(name='c'),
98 'd': mock.Mock(name='d'),
99 },
100 }
101 mock_extman = self.setup_extension_manager(extensions)
102
103 orig_iteritems = six.iteritems
104 iteritems_patch = mock.patch('six.iteritems')
105 mock_iteritems = iteritems_patch.start()
106 self.addCleanup(iteritems_patch.stop)
107
108 def sorted_iteritems(d):
109 return sorted(orig_iteritems(d), reverse=True)
110
111 mock_iteritems.side_effect = sorted_iteritems
112
113 res = transformations.Manager.load_transformers('testname', config)
114
115 self.assertEqual(res, [
116 (version.StrictVersion('8.0'), [
117 extensions['8.0']['c'],
118 ]),
119 (version.StrictVersion('9.0'), [
120 extensions['9.0']['a'],
121 extensions['9.0']['b'],
122 ]),
123 ])
124 callback = transformations.reraise_endpoint_load_failure
125 self.assertItemsEqual(mock_extman.mock_calls, [
126 mock.call(
127 'nailgun.cluster_upgrade.transformations.testname.9.0',
128 on_load_failure_callback=callback,
129 ),
130 mock.call(
131 'nailgun.cluster_upgrade.transformations.testname.8.0',
132 on_load_failure_callback=callback,
133 ),
134 ])
135
136 def test_load_transformers_keyerror(self):
137 config = {'9.0': ['a', 'b', 'c']}
138 extensions = {'9.0': {
139 'a': mock.Mock(name='a'),
140 'b': mock.Mock(name='b'),
141 }}
142 mock_extman = self.setup_extension_manager(extensions)
143
144 with self.assertRaisesRegexp(KeyError, 'c'):
145 transformations.Manager.load_transformers('testname', config)
146
147 callback = transformations.reraise_endpoint_load_failure
148 self.assertEqual(mock_extman.mock_calls, [
149 mock.call(
150 'nailgun.cluster_upgrade.transformations.testname.9.0',
151 on_load_failure_callback=callback,
152 ),
153 ])
154
155 @mock.patch.object(transformations.Manager, 'load_transformers')
156 def test_apply(self, mock_load):
157 mock_trans = mock.Mock()
158 mock_load.return_value = [
159 (version.StrictVersion('7.0'), [mock_trans.a, mock_trans.b]),
160 (version.StrictVersion('8.0'), [mock_trans.c, mock_trans.d]),
161 (version.StrictVersion('9.0'), [mock_trans.e, mock_trans.f]),
162 ]
163 man = transformations.Manager()
164 res = man.apply('7.0', '9.0', {})
165 self.assertEqual(res, mock_trans.f.return_value)
166 self.assertEqual(mock_trans.mock_calls, [
167 mock.call.c({}),
168 mock.call.d(mock_trans.c.return_value),
169 mock.call.e(mock_trans.d.return_value),
170 mock.call.f(mock_trans.e.return_value),
171 ])
172
173
174class TestLazy(nailgun_test_base.BaseUnitTest):
175 def test_lazy(self):
176 mgr_cls_mock = mock.Mock()
177 lazy_obj = transformations.Lazy(mgr_cls_mock)
178 lazy_obj.apply()
179 self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply)
diff --git a/cluster_upgrade/transformations/__init__.py b/cluster_upgrade/transformations/__init__.py
new file mode 100644
index 0000000..64c14d7
--- /dev/null
+++ b/cluster_upgrade/transformations/__init__.py
@@ -0,0 +1,94 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import copy
14import distutils.version
15import logging
16import threading
17
18import six
19
20import stevedore
21
22LOG = logging.getLogger(__name__)
23
24
25def reraise_endpoint_load_failure(manager, endpoint, exc):
26 LOG.error('Failed to load %s: %s', endpoint.name, exc)
27 raise # Avoid unexpectedly skipped steps
28
29
30class Manager(object):
31 default_config = None
32 name = None
33
34 def __init__(self):
35 self.config = self.get_config(self.name)
36 self.transformers = self.load_transformers(self.name, self.config)
37
38 @classmethod
39 def get_config(cls, name):
40 # TODO(yorik-sar): merge actual config with defaults
41 return cls.default_config
42
43 @staticmethod
44 def load_transformers(name, config):
45 transformers = []
46 for version, names in six.iteritems(config):
47 extension_manager = stevedore.ExtensionManager(
48 'nailgun.cluster_upgrade.transformations.{}.{}'.format(
49 name, version),
50 on_load_failure_callback=reraise_endpoint_load_failure,
51 )
52 try:
53 sorted_extensions = [extension_manager[n].plugin
54 for n in names]
55 except KeyError as exc:
56 LOG.error('%s transformer %s not found for version %s',
57 name, exc, version)
58 raise
59 strict_version = distutils.version.StrictVersion(version)
60 transformers.append((strict_version, sorted_extensions))
61 transformers.sort()
62 return transformers
63
64 def apply(self, from_version, to_version, data):
65 strict_from = distutils.version.StrictVersion(from_version)
66 strict_to = distutils.version.StrictVersion(to_version)
67 assert strict_from < strict_to, \
68 "from_version must be smaller than to_version"
69 data = copy.deepcopy(data)
70 for version, transformers in self.transformers:
71 if version <= strict_from:
72 continue
73 if version > strict_to:
74 break
75 for transformer in transformers:
76 LOG.debug("Applying %s transformer %s",
77 self.name, transformer)
78 data = transformer(data)
79 return data
80
81
82class Lazy(object):
83 def __init__(self, mgr_cls):
84 self.mgr_cls = mgr_cls
85 self.mgr = None
86 self.lock = threading.Lock()
87
88 def apply(self, *args, **kwargs):
89 if self.mgr is None:
90 with self.lock:
91 if self.mgr is None:
92 self.mgr = self.mgr_cls()
93 self.apply = self.mgr.apply
94 return self.mgr.apply(*args, **kwargs)