diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e8a886c --- /dev/null +++ b/README.rst @@ -0,0 +1,11 @@ +Fuel nailgun extenstion for cluster upgrade +=========================================== + +This extension for Nailgun provides API handlers and logic for +cluster upgrading. This extension used by the fuel-octane project. + +Instalation +----------- +After installing `fuel-nailgun-extension-cluster-upgrade` package run: +1) `nailgun_syncdb` - migrate database +2) restart nailgun service diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..568f269 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,6 @@ +libpq-dev +postgresql +postgresql-client +# We don't use these, but mysql-prep step is in template job +mysql-client +mysql-server diff --git a/cluster_upgrade/extension.py b/cluster_upgrade/extension.py index 66df106..79bdca4 100644 --- a/cluster_upgrade/extension.py +++ b/cluster_upgrade/extension.py @@ -33,6 +33,9 @@ class ClusterUpgradeExtension(extensions.BaseExtension): 'handler': handlers.NodeReassignHandler}, {'uri': r'/clusters/(?P\d+)/upgrade/vips/?$', 'handler': handlers.CopyVIPsHandler}, + {'uri': r'/clusters/(?P\d+)/upgrade/clone_release/' + r'(?P\d+)/?$', + 'handler': handlers.CreateUpgradeReleaseHandler}, ] @classmethod diff --git a/cluster_upgrade/handlers.py b/cluster_upgrade/handlers.py index 4f61f2e..d4b7f71 100644 --- a/cluster_upgrade/handlers.py +++ b/cluster_upgrade/handlers.py @@ -29,7 +29,9 @@ class ClusterUpgradeCloneHandler(base.BaseHandler): single = objects.Cluster validator = validators.ClusterUpgradeValidator - @base.content + @base.handle_errors + @base.validate + @base.serialize def POST(self, cluster_id): """Initialize the upgrade of the cluster. @@ -50,7 +52,7 @@ class ClusterUpgradeCloneHandler(base.BaseHandler): request_data = self.checked_data(cluster=orig_cluster) new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster, request_data) - return new_cluster.to_json() + return new_cluster.to_dict() class NodeReassignHandler(base.BaseHandler): @@ -67,7 +69,8 @@ class NodeReassignHandler(base.BaseHandler): self.raise_task(task) - @base.content + @base.handle_errors + @base.validate def POST(self, cluster_id): """Reassign node to the given cluster. @@ -107,7 +110,8 @@ class CopyVIPsHandler(base.BaseHandler): single = objects.Cluster validator = validators.CopyVIPsValidator - @base.content + @base.handle_errors + @base.validate def POST(self, cluster_id): """Copy VIPs from original cluster to new one @@ -139,3 +143,45 @@ class CopyVIPsHandler(base.BaseHandler): upgrade.UpgradeHelper.copy_vips(orig_cluster_adapter, seed_cluster_adapter) + + +class CreateUpgradeReleaseHandler(base.BaseHandler): + @staticmethod + def merge_network_roles(base_nets, orig_nets): + """Create network metadata based on two releases. + + Overwrite base default_mapping by orig default_maping values. + """ + orig_network_dict = {n['id']: n for n in orig_nets} + for base_net in base_nets: + orig_net = orig_network_dict.get(base_net['id']) + if orig_net is None: + orig_net = base_net + base_net['default_mapping'] = orig_net['default_mapping'] + return base_net + + @base.serialize + def POST(self, cluster_id, release_id): + """Create release for upgrade purposes. + + Creates a new release with network_roles_metadata based the given + release and re-use network parameters from the given cluster. + + :returns: JSON representation of the created cluster + :http: * 200 (OK) + * 404 (Cluster or release not found.) + """ + base_release = self.get_object_or_404(objects.Release, release_id) + orig_cluster = self.get_object_or_404(objects.Cluster, cluster_id) + orig_release = orig_cluster.release + + network_metadata = self.merge_network_roles( + base_release.network_roles_metadata, + orig_release.network_roles_metadata) + data = objects.Release.to_dict(base_release) + data['network_roles_metadata'] = network_metadata + data['name'] = '{0} Upgrade ({1})'.format( + base_release.name, orig_release.id) + del data['id'] + new_release = objects.Release.create(data) + return new_release.to_dict() diff --git a/cluster_upgrade/objects/adapters.py b/cluster_upgrade/objects/adapters.py index 7a2854e..d3b00e4 100644 --- a/cluster_upgrade/objects/adapters.py +++ b/cluster_upgrade/objects/adapters.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nailgun.extensions.volume_manager import extension as volume_ext from nailgun import objects @@ -62,6 +63,14 @@ class NailgunClusterAdapter(object): def editable_attrs(self, attrs): self.cluster.attributes.editable = attrs + @property + def network_template(self): + return self.cluster.network_config.configuration_template + + @network_template.setter + def network_template(self, template): + self.cluster.network_config.configuration_template = template + def get_create_data(self): return objects.Cluster.get_create_data(self.cluster) @@ -70,8 +79,8 @@ class NailgunClusterAdapter(object): instance=self.cluster) return NailgunNetworkManager(self.cluster, net_manager) - def to_json(self): - return objects.Cluster.to_json(self.cluster) + def to_dict(self): + return objects.Cluster.to_dict(self.cluster) @classmethod def get_by_uid(cls, cluster_id): @@ -96,6 +105,10 @@ class NailgunReleaseAdapter(object): uid, fail_if_not_found=fail_if_not_found) return release + @property + def operating_system(self): + return self.release.operating_system + @property def is_deployable(self): return objects.Release.is_deployable(self.release) @@ -173,6 +186,10 @@ class NailgunNodeAdapter(object): def status(self): return self.node.status + @property + def nic_interfaces(self): + return self.node.nic_interfaces + @property def error_type(self): return self.node.error_type @@ -192,6 +209,14 @@ class NailgunNodeAdapter(object): def add_pending_change(self, change): objects.Node.add_pending_change(self.node, change) + def get_volumes(self): + return volume_ext.VolumeManagerExtension.get_node_volumes(self.node) + + def set_volumes(self, volumes): + return volume_ext.VolumeManagerExtension.set_node_volumes( + self.node, volumes + ) + class NailgunNetworkGroupAdapter(object): diff --git a/cluster_upgrade/tests/test_handlers.py b/cluster_upgrade/tests/test_handlers.py index 336b9d4..ad7239a 100644 --- a/cluster_upgrade/tests/test_handlers.py +++ b/cluster_upgrade/tests/test_handlers.py @@ -77,12 +77,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): @mock.patch('nailgun.task.task.rpc.cast') def test_node_reassign_handler(self, mcast): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': consts.NODE_STATUSES.ready}]) - self.env.create_cluster() - cluster = self.env.clusters[0] - seed_cluster = self.env.clusters[1] + seed_cluster = self.env.create_cluster() node_id = cluster.nodes[0]['id'] resp = self.app.post( @@ -144,9 +142,7 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): self.assertEqual(node.roles, ['compute']) def test_node_reassign_handler_no_node(self): - self.env.create_cluster() - - cluster = self.env.clusters[0] + cluster = self.env.create_cluster() resp = self.app.post( reverse('NodeReassignHandler', @@ -159,10 +155,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): resp.json_body['message']) def test_node_reassing_handler_wrong_status(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'discover'}]) - cluster = self.env.clusters[0] resp = self.app.post( reverse('NodeReassignHandler', @@ -175,11 +170,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): "^Node should be in one of statuses:") def test_node_reassing_handler_wrong_error_type(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'error', 'error_type': 'provision'}]) - cluster = self.env.clusters[0] resp = self.app.post( reverse('NodeReassignHandler', @@ -192,10 +186,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest): "^Node should be in error state") def test_node_reassign_handler_to_the_same_cluster(self): - self.env.create( + cluster = self.env.create( cluster_kwargs={'api': False}, nodes_kwargs=[{'status': 'ready'}]) - cluster = self.env.clusters[0] cluster_id = cluster['id'] node_id = cluster.nodes[0]['id'] diff --git a/cluster_upgrade/tests/test_transformations.py b/cluster_upgrade/tests/test_transformations.py new file mode 100644 index 0000000..880ee2d --- /dev/null +++ b/cluster_upgrade/tests/test_transformations.py @@ -0,0 +1,221 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from distutils import version + +import mock +from nailgun.test import base as nailgun_test_base +import six + +from .. import transformations +from ..transformations import cluster + + +class TestTransformations(nailgun_test_base.BaseUnitTest): + def test_get_config(self): + config = object() + + class Manager(transformations.Manager): + default_config = config + + self.assertIs(config, Manager.get_config('testname')) + + def setup_extension_manager(self, extensions): + p = mock.patch("stevedore.ExtensionManager", spec=['__call__']) + mock_extman = p.start() + self.addCleanup(p.stop) + + def extman(namespace, *args, **kwargs): + instance = mock.MagicMock(name=namespace) + ext_results = {} + for ver, exts in six.iteritems(extensions): + if namespace.endswith(ver): + ext_results = {name: mock.Mock(name=name, plugin=ext) + for name, ext in six.iteritems(exts)} + break + else: + self.fail("Called with unexpected version in namespace: {}, " + "expected versions: {}".format( + namespace, list(extensions))) + instance.__getitem__.side_effect = ext_results.__getitem__ + return instance + + mock_extman.side_effect = extman + return mock_extman + + def test_load_transformers(self): + config = {'9.0': ['a', 'b']} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, [(version.StrictVersion('9.0'), [ + extensions['9.0']['a'], + extensions['9.0']['b'], + ])]) + callback = transformations.reraise_endpoint_load_failure + self.assertEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + ]) + + def test_load_transformers_empty(self): + config = {} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, []) + self.assertEqual(mock_extman.mock_calls, []) + + def test_load_transformers_sorted(self): + config = {'9.0': ['a', 'b'], '8.0': ['c']} + extensions = { + '9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }, + '8.0': { + 'c': mock.Mock(name='c'), + 'd': mock.Mock(name='d'), + }, + } + mock_extman = self.setup_extension_manager(extensions) + + orig_iteritems = six.iteritems + iteritems_patch = mock.patch('six.iteritems') + mock_iteritems = iteritems_patch.start() + self.addCleanup(iteritems_patch.stop) + + def sorted_iteritems(d): + return sorted(orig_iteritems(d), reverse=True) + + mock_iteritems.side_effect = sorted_iteritems + + res = transformations.Manager.load_transformers('testname', config) + + self.assertEqual(res, [ + (version.StrictVersion('8.0'), [ + extensions['8.0']['c'], + ]), + (version.StrictVersion('9.0'), [ + extensions['9.0']['a'], + extensions['9.0']['b'], + ]), + ]) + callback = transformations.reraise_endpoint_load_failure + self.assertItemsEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.8.0', + on_load_failure_callback=callback, + ), + ]) + + def test_load_transformers_keyerror(self): + config = {'9.0': ['a', 'b', 'c']} + extensions = {'9.0': { + 'a': mock.Mock(name='a'), + 'b': mock.Mock(name='b'), + }} + mock_extman = self.setup_extension_manager(extensions) + + with self.assertRaisesRegexp(KeyError, 'c'): + transformations.Manager.load_transformers('testname', config) + + callback = transformations.reraise_endpoint_load_failure + self.assertEqual(mock_extman.mock_calls, [ + mock.call( + 'nailgun.cluster_upgrade.transformations.testname.9.0', + on_load_failure_callback=callback, + ), + ]) + + @mock.patch.object(transformations.Manager, 'load_transformers') + def test_apply(self, mock_load): + mock_trans = mock.Mock() + mock_load.return_value = [ + (version.StrictVersion('7.0'), [mock_trans.a, mock_trans.b]), + (version.StrictVersion('8.0'), [mock_trans.c, mock_trans.d]), + (version.StrictVersion('9.0'), [mock_trans.e, mock_trans.f]), + ] + man = transformations.Manager() + res = man.apply('7.0', '9.0', {}) + self.assertEqual(res, mock_trans.f.return_value) + self.assertEqual(mock_trans.mock_calls, [ + mock.call.c({}), + mock.call.d(mock_trans.c.return_value), + mock.call.e(mock_trans.d.return_value), + mock.call.f(mock_trans.e.return_value), + ]) + + +class TestLazy(nailgun_test_base.BaseUnitTest): + def test_lazy(self): + mgr_cls_mock = mock.Mock() + lazy_obj = transformations.Lazy(mgr_cls_mock) + lazy_obj.apply() + self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply) + + +class TestClusterTransformers(nailgun_test_base.BaseUnitTest): + def setUp(self): + self.data = { + 'editable': { + 'external_dns': { + 'dns_list': {'type': 'text', 'value': 'a,b,\nc, d'}}, + 'external_ntp': { + 'ntp_list': {'type': 'text', 'value': 'a,b,\nc, d'}}, + }, + 'generated': { + 'provision': {}, + }, + } + + def test_dns_list(self): + res = cluster.transform_dns_list(self.data) + self.assertEqual( + res['editable']['external_dns']['dns_list'], + {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']}, + ) + + def test_ntp_list(self): + res = cluster.transform_ntp_list(self.data) + self.assertEqual( + res['editable']['external_ntp']['ntp_list'], + {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']}, + ) + + def test_provision(self): + res = cluster.drop_generated_provision(self.data) + self.assertNotIn('provision', res['generated']) + + def test_manager(self): + man = cluster.Manager() # verify default config and entry points + self.assertEqual(man.transformers, [(version.StrictVersion('9.0'), [ + cluster.transform_dns_list, + cluster.transform_ntp_list, + cluster.drop_generated_provision, + ])]) diff --git a/cluster_upgrade/tests/test_upgrade.py b/cluster_upgrade/tests/test_upgrade.py index 296d819..ef7e13e 100644 --- a/cluster_upgrade/tests/test_upgrade.py +++ b/cluster_upgrade/tests/test_upgrade.py @@ -19,6 +19,7 @@ import six from nailgun import consts from nailgun.objects.serializers import network_configuration +from nailgun.test.base import fake_tasks from .. import upgrade from . import base as base_tests @@ -49,7 +50,7 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): {"metadata": "src_fake", "key": {"type": "text", - "value": "fake1, fake2,fake3 , fake4"}, + "value": "fake"}, "src_key": "src_data" }, "repo_setup": "src_data" @@ -68,9 +69,6 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): result = upgrade.merge_attributes( src_editable_attrs, new_editable_attrs ) - new_editable_attrs["test"]["key"]["value"] = [ - "fake1", "fake2", "fake3", "fake4" - ] self.assertEqual(result, new_editable_attrs) def test_create_cluster_clone(self): @@ -238,3 +236,44 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest): self.helper.change_env_settings(self.src_cluster, new_cluster) self.assertEqual('image', attrs['editable']['provision']['method']['value']) + + def get_assigned_nets(self, node): + assigned_nets = {} + for iface in node.nic_interfaces: + nets = [net.name for net in iface.assigned_networks_list] + assigned_nets[iface.name] = nets + return assigned_nets + + @fake_tasks() + def assign_node_to_cluster(self, template=None): + new_cluster = self.helper.clone_cluster(self.src_cluster, self.data) + node = adapters.NailgunNodeAdapter(self.src_cluster.cluster.nodes[0]) + + orig_assigned_nets = self.get_assigned_nets(node) + + if template: + net_template = self.env.read_fixtures(['network_template_80'])[0] + new_cluster.network_template = net_template + orig_assigned_nets = { + 'eth0': ['fuelweb_admin'], 'eth1': ['public', 'management'] + } + + self.helper.assign_node_to_cluster(node, new_cluster, node.roles, []) + self.db.refresh(new_cluster.cluster) + + self.assertEqual(node.cluster_id, new_cluster.id) + + self.env.clusters.append(new_cluster.cluster) + task = self.env.launch_provisioning_selected(cluster_id=new_cluster.id) + self.assertEqual(task.status, consts.TASK_STATUSES.ready) + for n in new_cluster.cluster.nodes: + self.assertEqual(consts.NODE_STATUSES.provisioned, n.status) + + new_assigned_nets = self.get_assigned_nets(node) + self.assertEqual(orig_assigned_nets, new_assigned_nets) + + def test_assign_node_to_cluster(self): + self.assign_node_to_cluster() + + def test_assign_node_to_cluster_with_template(self): + self.assign_node_to_cluster(template=True) diff --git a/cluster_upgrade/tests/test_validators.py b/cluster_upgrade/tests/test_validators.py index 4c7c964..43da11c 100644 --- a/cluster_upgrade/tests/test_validators.py +++ b/cluster_upgrade/tests/test_validators.py @@ -58,6 +58,14 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest): self.validator.validate_release_upgrade(self.dst_release, self.src_release) + def test_validate_release_upgrade_to_different_os(self): + self.dst_release.operating_system = consts.RELEASE_OS.centos + msg = "^Changing of operating system is not possible during upgrade " \ + "\(from {0} to {1}\).$".format("Ubuntu", "CentOS") + with self.assertRaisesRegexp(errors.InvalidData, msg): + self.validator.validate_release_upgrade(self.src_release, + self.dst_release) + def test_validate_cluster_name(self): self.validator.validate_cluster_name("cluster-42") @@ -187,10 +195,14 @@ class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest): "reprovision": False, "roles": ['controller', 'compute'], }) - msg = "^Role 'controller' in conflict with role 'compute'.$" - with self.assertRaisesRegexp(errors.InvalidData, msg): + with self.assertRaises(errors.InvalidData) as exc: self.validator.validate(data, self.dst_cluster) + self.assertEqual( + exc.exception.message, + "Role 'controller' in conflict with role 'compute'." + ) + class TestCopyVIPsValidator(base.BaseTestCase): validator = validators.CopyVIPsValidator diff --git a/cluster_upgrade/transformations/__init__.py b/cluster_upgrade/transformations/__init__.py new file mode 100644 index 0000000..7e1d861 --- /dev/null +++ b/cluster_upgrade/transformations/__init__.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import distutils.version +import logging +import threading + +import six + +import stevedore + +LOG = logging.getLogger(__name__) + + +def reraise_endpoint_load_failure(manager, endpoint, exc): + LOG.error('Failed to load %s: %s', endpoint.name, exc) + raise # Avoid unexpectedly skipped steps + + +class Manager(object): + default_config = None + name = None + + def __init__(self): + self.config = self.get_config(self.name) + self.transformers = self.load_transformers(self.name, self.config) + + @classmethod + def get_config(cls, name): + # TODO(yorik-sar): merge actual config with defaults + return cls.default_config + + @staticmethod + def load_transformers(name, config): + transformers = [] + for version, names in six.iteritems(config): + extension_manager = stevedore.ExtensionManager( + 'nailgun.cluster_upgrade.transformations.{}.{}'.format( + name, version), + on_load_failure_callback=reraise_endpoint_load_failure, + ) + try: + sorted_extensions = [extension_manager[n].plugin + for n in names] + except KeyError as exc: + LOG.error('%s transformer %s not found for version %s', + name, exc, version) + raise + strict_version = distutils.version.StrictVersion(version) + transformers.append((strict_version, sorted_extensions)) + transformers.sort() + return transformers + + def apply(self, from_version, to_version, data): + strict_from = distutils.version.StrictVersion(from_version) + strict_to = distutils.version.StrictVersion(to_version) + assert strict_from <= strict_to, \ + "from_version must not be greater than to_version" + data = copy.deepcopy(data) + for version, transformers in self.transformers: + if version <= strict_from: + continue + if version > strict_to: + break + for transformer in transformers: + LOG.debug("Applying %s transformer %s", + self.name, transformer) + data = transformer(data) + return data + + +class Lazy(object): + def __init__(self, mgr_cls): + self.mgr_cls = mgr_cls + self.mgr = None + self.lock = threading.Lock() + + def apply(self, *args, **kwargs): + if self.mgr is None: + with self.lock: + if self.mgr is None: + self.mgr = self.mgr_cls() + self.apply = self.mgr.apply + return self.mgr.apply(*args, **kwargs) diff --git a/cluster_upgrade/transformations/cluster.py b/cluster_upgrade/transformations/cluster.py new file mode 100644 index 0000000..61368e0 --- /dev/null +++ b/cluster_upgrade/transformations/cluster.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cluster_upgrade import transformations + +# NOTE: In the mitaka-9.0 release types of values dns_list and +# ntp_list were changed from 'text' +# (a string of comma-separated IP-addresses) +# to 'text_list' (a list of strings of IP-addresses). + + +def transform_to_text_list(data): + if data['type'] == 'text': + data['type'] = 'text_list' + data['value'] = [ + part.strip() for part in data['value'].split(',') + ] + + return data + + +def transform_dns_list(data): + dns_list = data['editable']['external_dns']['dns_list'] + transform_to_text_list(dns_list) + return data + + +def transform_ntp_list(data): + ntp_list = data['editable']['external_ntp']['ntp_list'] + transform_to_text_list(ntp_list) + return data + + +def drop_generated_provision(data): + data['generated'].pop('provision', None) + return data + + +class Manager(transformations.Manager): + default_config = { + '9.0': ['dns_list', 'ntp_list', 'drop_provision'], + } + name = 'cluster' diff --git a/cluster_upgrade/transformations/vip.py b/cluster_upgrade/transformations/vip.py new file mode 100644 index 0000000..417b5f6 --- /dev/null +++ b/cluster_upgrade/transformations/vip.py @@ -0,0 +1,62 @@ +# coding: utf-8 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +from cluster_upgrade import transformations + + +def transform_vips(data): + """Rename or remove types of VIPs for 7.0 network groups. + + This method renames types of VIPs from older releases (<7.0) to + be compatible with network groups of the 7.0 release according + to the rules: + + management: haproxy -> management + public: haproxy -> public + public: vrouter -> vrouter_pub + + Note, that in the result VIPs are present only those IPs that + correspond to the given rules. + """ + rename_vip_rules = { + "management": { + "haproxy": "management", + "vrouter": "vrouter", + }, + "public": { + "haproxy": "public", + "vrouter": "vrouter_pub", + }, + } + renamed_vips = collections.defaultdict(dict) + for ng_name, vips_obj in data.items(): + + ng_vip_rules = rename_vip_rules[ng_name] + for vip_name, vip_addr in vips_obj.items(): + if vip_name not in ng_vip_rules: + continue + + new_vip_name = ng_vip_rules[vip_name] + renamed_vips[ng_name][new_vip_name] = vip_addr + + return renamed_vips + + +class Manager(transformations.Manager): + default_config = { + '7.0': ['transform_vips'] + } + name = 'vip' diff --git a/cluster_upgrade/transformations/volumes.py b/cluster_upgrade/transformations/volumes.py new file mode 100644 index 0000000..cb9dca7 --- /dev/null +++ b/cluster_upgrade/transformations/volumes.py @@ -0,0 +1,53 @@ +# coding: utf-8 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from cluster_upgrade import transformations + + +def transform_node_volumes(volumes): + try: + os_vg = next(vol for vol in volumes + if 'id' in vol and vol['id'] == 'os') + except StopIteration: + return volumes + + other_volumes = [vol for vol in volumes + if 'id' not in vol or vol['id'] != 'os'] + + for disk in other_volumes: + disk_volumes = disk['volumes'] + disk['volumes'] = [] + + for v in disk_volumes: + if v['type'] == 'pv' and v['vg'] == 'os' and v['size'] > 0: + for vv in os_vg['volumes']: + partition = {'name': vv['name'], + 'size': vv['size'], + 'type': 'partition', + 'mount': vv['mount'], + 'file_system': vv['file_system']} + disk['volumes'].append(partition) + else: + if v['type'] == 'lvm_meta_pool' or v['type'] == 'boot': + v['size'] = 0 + disk['volumes'].append(v) + + return volumes + + +class Manager(transformations.Manager): + default_config = { + '6.1': ['node_volumes'] + } + name = 'volumes' diff --git a/cluster_upgrade/upgrade.py b/cluster_upgrade/upgrade.py index 939cc4a..52c1024 100644 --- a/cluster_upgrade/upgrade.py +++ b/cluster_upgrade/upgrade.py @@ -14,9 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import copy -from distutils import version import six from nailgun import consts @@ -24,7 +22,11 @@ from nailgun import objects from nailgun.objects.serializers import network_configuration from nailgun import utils +from . import transformations # That's weird, but that's how hacking likes from .objects import adapters +from .transformations import cluster as cluster_trs +from .transformations import vip +from .transformations import volumes as volumes_trs def merge_attributes(a, b): @@ -41,25 +43,9 @@ def merge_attributes(a, b): for key, values in six.iteritems(pairs): if key != "metadata" and key in a_values: values["value"] = a_values[key]["value"] - # NOTE: In the mitaka-9.0 release types of values dns_list and - # ntp_list were changed from 'text' - # (a string of comma-separated IP-addresses) - # to 'text_list' (a list of strings of IP-addresses). - if a_values[key]['type'] == 'text' and \ - values['type'] == 'text_list': - values["value"] = [ - value.strip() for value in values['value'].split(',') - ] return attrs -def merge_generated_attrs(new_attrs, orig_attrs): - # skip attributes that should be generated for new cluster - attrs = copy.deepcopy(orig_attrs) - attrs.pop('provision', None) - return utils.dict_merge(new_attrs, attrs) - - def merge_nets(a, b): new_settings = copy.deepcopy(b) source_networks = dict((n["name"], n) for n in a["networks"]) @@ -87,6 +73,9 @@ class UpgradeHelper(object): consts.CLUSTER_NET_PROVIDERS.nova_network: network_configuration.NovaNetworkConfigurationSerializer, } + cluster_transformations = transformations.Lazy(cluster_trs.Manager) + vip_transformations = transformations.Lazy(vip.Manager) + volumes_transformations = transformations.Lazy(volumes_trs.Manager) @classmethod def clone_cluster(cls, orig_cluster, data): @@ -110,61 +99,30 @@ class UpgradeHelper(object): @classmethod def copy_attributes(cls, orig_cluster, new_cluster): - # TODO(akscram): Attributes should be copied including - # borderline cases when some parameters are - # renamed or moved into plugins. Also, we should - # to keep special steps in copying of parameters - # that know how to translate parameters from one - # version to another. A set of this kind of steps - # should define an upgrade path of a particular - # cluster. - new_cluster.generated_attrs = merge_generated_attrs( + attrs = cls.cluster_transformations.apply( + orig_cluster.release.environment_version, + new_cluster.release.environment_version, + { + 'editable': orig_cluster.editable_attrs, + 'generated': orig_cluster.generated_attrs, + }, + ) + + new_cluster.generated_attrs = utils.dict_merge( new_cluster.generated_attrs, - orig_cluster.generated_attrs) + attrs['generated'], + ) + new_cluster.editable_attrs = merge_attributes( - orig_cluster.editable_attrs, - new_cluster.editable_attrs) + attrs['editable'], + new_cluster.editable_attrs, + ) @classmethod def change_env_settings(cls, orig_cluster, new_cluster): attrs = new_cluster.attributes attrs['editable']['provision']['method']['value'] = 'image' - @classmethod - def transform_vips_for_net_groups_70(cls, vips): - """Rename or remove types of VIPs for 7.0 network groups. - - This method renames types of VIPs from older releases (<7.0) to - be compatible with network groups of the 7.0 release according - to the rules: - - management: haproxy -> management - public: haproxy -> public - public: vrouter -> vrouter_pub - - Note, that in the result VIPs are present only those IPs that - correspond to the given rules. - """ - rename_vip_rules = { - "management": { - "haproxy": "management", - "vrouter": "vrouter", - }, - "public": { - "haproxy": "public", - "vrouter": "vrouter_pub", - }, - } - renamed_vips = collections.defaultdict(dict) - for ng_name, vips in six.iteritems(vips): - ng_vip_rules = rename_vip_rules[ng_name] - for vip_name, vip_addr in six.iteritems(vips): - if vip_name not in ng_vip_rules: - continue - new_vip_name = ng_vip_rules[vip_name] - renamed_vips[ng_name][new_vip_name] = vip_addr - return renamed_vips - @classmethod def copy_network_config(cls, orig_cluster, new_cluster): nets_serializer = cls.network_serializers[orig_cluster.net_provider] @@ -181,17 +139,16 @@ class UpgradeHelper(object): orig_net_manager = orig_cluster.get_network_manager() new_net_manager = new_cluster.get_network_manager() - vips = orig_net_manager.get_assigned_vips() - for ng_name in vips: - if ng_name not in (consts.NETWORKS.public, - consts.NETWORKS.management): - vips.pop(ng_name) - # NOTE(akscram): In the 7.0 release was introduced networking - # templates that use the vip_name column as - # unique names of VIPs. - if version.LooseVersion(orig_cluster.release.environment_version) < \ - version.LooseVersion("7.0"): - vips = cls.transform_vips_for_net_groups_70(vips) + vips = {} + assigned_vips = orig_net_manager.get_assigned_vips() + for ng_name in (consts.NETWORKS.public, consts.NETWORKS.management): + vips[ng_name] = assigned_vips[ng_name] + + vips = cls.vip_transformations.apply( + orig_cluster.release.environment_version, + new_cluster.release.environment_version, + vips + ) new_net_manager.assign_given_vips_for_net_groups(vips) new_net_manager.assign_vips_for_net_groups() @@ -224,6 +181,13 @@ class UpgradeHelper(object): orig_cluster = adapters.NailgunClusterAdapter.get_by_uid( node.cluster_id) + volumes = cls.volumes_transformations.apply( + orig_cluster.release.environment_version, + seed_cluster.release.environment_version, + node.get_volumes(), + ) + node.set_volumes(volumes) + orig_manager = orig_cluster.get_network_manager() netgroups_id_mapping = cls.get_netgroups_id_mapping( @@ -231,10 +195,13 @@ class UpgradeHelper(object): node.update_cluster_assignment(seed_cluster, roles, pending_roles) objects.Node.set_netgroups_ids(node, netgroups_id_mapping) - orig_manager.set_nic_assignment_netgroups_ids( - node, netgroups_id_mapping) - orig_manager.set_bond_assignment_netgroups_ids( - node, netgroups_id_mapping) + + if not seed_cluster.network_template: + orig_manager.set_nic_assignment_netgroups_ids( + node, netgroups_id_mapping) + orig_manager.set_bond_assignment_netgroups_ids( + node, netgroups_id_mapping) + node.add_pending_change(consts.CLUSTER_CHANGES.interfaces) @classmethod diff --git a/cluster_upgrade/validators.py b/cluster_upgrade/validators.py index 3874147..95f2f6d 100644 --- a/cluster_upgrade/validators.py +++ b/cluster_upgrade/validators.py @@ -62,6 +62,12 @@ class ClusterUpgradeValidator(base.BasicValidator): "this release is equal or lower than the release of the " "original cluster.".format(new_release.id), log_message=True) + if orig_release.operating_system != new_release.operating_system: + raise errors.InvalidData( + "Changing of operating system is not possible during upgrade " + "(from {0} to {1}).".format(orig_release.operating_system, + new_release.operating_system), + log_message=True) @classmethod def validate_cluster_name(cls, cluster_name): diff --git a/setup.cfg b/setup.cfg index 3d6beb9..c88b57f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [metadata] name = fuel-nailgun-extension-cluster-upgrade summary = Cluster upgrade extension for Fuel +description-file = README.rst author = Mirantis Inc. author-email = product@mirantis.com home-page = http://mirantis.com @@ -24,3 +25,11 @@ packages = [entry_points] nailgun.extensions = cluster_upgrade = cluster_upgrade.extension:ClusterUpgradeExtension +nailgun.cluster_upgrade.transformations.volumes.6.1 = + node_volumes = cluster_upgrade.transformations.volumes:transform_node_volumes +nailgun.cluster_upgrade.transformations.cluster.9.0 = + dns_list = cluster_upgrade.transformations.cluster:transform_dns_list + ntp_list = cluster_upgrade.transformations.cluster:transform_ntp_list + drop_provision = cluster_upgrade.transformations.cluster:drop_generated_provision +nailgun.cluster_upgrade.transformations.vip.7.0 = + transform_vips = cluster_upgrade.transformations.vip:transform_vips