From 5358702c26f6ed6eb0db1df0fbe661bc65f20c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Thu, 27 Sep 2018 10:20:51 +0200 Subject: [PATCH] Add kuryr-status utility for upgrade-checkers This commit adds kuryr-status utility that can be used to check if upgrade is possible, convert annotations to new format and rollback those changes if needed. Implements: blueprint upgrade-checkers Change-Id: I7a40a68518d7fbba18146b64befb6f585176ec8d --- doc/source/installation/index.rst | 1 + doc/source/installation/upgrades.rst | 89 ++++++ kuryr_kubernetes/cmd/status.py | 266 ++++++++++++++++++ .../tests/unit/cmd/test_status.py | 183 ++++++++++++ kuryr_kubernetes/version.py | 3 +- .../notes/stein-upgrade-226c8e7b735701ee.yaml | 6 + requirements.txt | 1 + setup.cfg | 1 + 8 files changed, 548 insertions(+), 2 deletions(-) create mode 100644 doc/source/installation/upgrades.rst create mode 100644 kuryr_kubernetes/cmd/status.py create mode 100644 kuryr_kubernetes/tests/unit/cmd/test_status.py create mode 100644 releasenotes/notes/stein-upgrade-226c8e7b735701ee.yaml diff --git a/doc/source/installation/index.rst b/doc/source/installation/index.rst index 5c56304bb..e0b5d6cfb 100644 --- a/doc/source/installation/index.rst +++ b/doc/source/installation/index.rst @@ -33,6 +33,7 @@ This section describes how you can install and configure kuryr-kubernetes ports-pool services ipv6 + upgrades devstack/index default_configuration trunk_ports diff --git a/doc/source/installation/upgrades.rst b/doc/source/installation/upgrades.rst new file mode 100644 index 000000000..8ed4e9584 --- /dev/null +++ b/doc/source/installation/upgrades.rst @@ -0,0 +1,89 @@ +Upgrading kuryr-kubernetes +=========================== + +Kuryr-Kubernetes supports standard OpenStack utility for checking upgrade +is possible and safe: + +.. code-block:: bash + + $ kuryr-status upgrade check + +---------------------------------------+ + | Upgrade Check Results | + +---------------------------------------+ + | Check: Pod annotations | + | Result: Success | + | Details: All annotations are updated. | + +---------------------------------------+ + +If any issue will be found, the utility will give you explanation and possible +remediations. Also note that *Warning* results aren't blocking an upgrade, but +are worth investigating. + +Stein (0.6.x) to T (0.7.x) upgrade +---------------------------------- + +In T we want to drop support for old format of Pod annotations (switch was +motivated by multi-vif support feature implemented in Rocky). To make sure that +you don't have unsupported Pod annotations you need to run ``kuryr-status +upgrade check`` utility **before upgrading Kuryr-Kubernetes services to T**. + +.. note:: + + In case of running Kuryr-Kubernetes containerized you can use ``kubectl + exec`` to run kuryr-status + + .. code-block:: bash + + $ kubectl -n kube-system exec -it kuryr-status upgrade check + +.. code-block:: bash + + $ kuryr-status upgrade check + +---------------------------------------+ + | Upgrade Check Results | + +---------------------------------------+ + | Check: Pod annotations | + | Result: Success | + | Details: All annotations are updated. | + +---------------------------------------+ + +In case of *Failure* result of *Pod annotations* check you should run +``kuryr-status upgrade update-annotations`` command and check again: + +.. code-block:: bash + + $ kuryr-status upgrade check + +----------------------------------------------------------------------+ + | Upgrade Check Results | + +----------------------------------------------------------------------+ + | Check: Pod annotations | + | Result: Failure | + | Details: You have 3 Kuryr pod annotations in old format. You need to | + | run `kuryr-status upgrade update-annotations` | + | before proceeding with the upgrade. | + +----------------------------------------------------------------------+ + $ kuryr-status upgrade update-annotations + +-----------------------+--------+ + | Stat | Number | + +-----------------------+--------+ + | Updated annotations | 3 | + +-----------------------+--------+ + | Malformed annotations | 0 | + +-----------------------+--------+ + | Annotations left | 0 | + +-----------------------+--------+ + $ kuryr-status upgrade check + +---------------------------------------+ + | Upgrade Check Results | + +---------------------------------------+ + | Check: Pod annotations | + | Result: Success | + | Details: All annotations are updated. | + +---------------------------------------+ + +It's possible that some annotations were somehow malformed. That will generate +a warning that should be investigated, but isn't blocking upgrading to T +(it won't make things any worse). + +If in any case you need to rollback those changes, there is +``kuryr-status upgrade downgrade-annotations`` command as well. diff --git a/kuryr_kubernetes/cmd/status.py b/kuryr_kubernetes/cmd/status.py new file mode 100644 index 000000000..b8553daa8 --- /dev/null +++ b/kuryr_kubernetes/cmd/status.py @@ -0,0 +1,266 @@ +# Copyright 2018 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +CLI interface for kuryr status commands. +""" + +from __future__ import print_function + +import sys +import textwrap +import traceback + +import prettytable + +import os_vif +from os_vif.objects import base +from oslo_config import cfg +from oslo_serialization import jsonutils + +from kuryr_kubernetes import clients +from kuryr_kubernetes import config +from kuryr_kubernetes import constants +from kuryr_kubernetes import exceptions +from kuryr_kubernetes import objects +from kuryr_kubernetes.objects import vif +from kuryr_kubernetes import version + +CONF = config.CONF + +UPGRADE_CHECK_SUCCESS = 0 +UPGRADE_CHECK_WARNING = 1 +UPGRADE_CHECK_FAILURE = 2 + +UPGRADE_CHECK_MSG_MAP = { + UPGRADE_CHECK_SUCCESS: 'Success', + UPGRADE_CHECK_WARNING: 'Warning', + UPGRADE_CHECK_FAILURE: 'Failure', +} + + +class UpgradeCheckResult(object): + """Class used for 'kuryr-status upgrade check' results. + + The 'code' attribute is an UpgradeCheckCode enum. + The 'details' attribute is a message generally only used for + checks that result in a warning or failure code. The details should provide + information on what issue was discovered along with any remediation. + """ + + def __init__(self, code, details=None): + super(UpgradeCheckResult, self).__init__() + self.code = code + self.details = details + + def get_details(self): + if self.details is not None: + # wrap the text on the details to 60 characters + return '\n'.join(textwrap.wrap(self.details, 60, + subsequent_indent=' ' * 9)) + + +class UpgradeCommands(object): + def __init__(self): + self.check_methods = { + 'Pod annotations': self._check_annotations, # Stein + } + clients.setup_kubernetes_client() + self.k8s = clients.get_kubernetes_client() + + def _get_annotation(self, pod): + annotations = pod['metadata']['annotations'] + if constants.K8S_ANNOTATION_VIF not in annotations: + # NOTE(dulek): We ignore pods without annotation, those + # probably are hostNetworking. + return None + k_ann = annotations[constants.K8S_ANNOTATION_VIF] + k_ann = jsonutils.loads(k_ann) + obj = base.VersionedObject.obj_from_primitive(k_ann) + return obj + + def _check_annotations(self): + old_count = 0 + malformed_count = 0 + pods = self.k8s.get('/api/v1/pods')['items'] + for pod in pods: + try: + obj = self._get_annotation(pod) + if not obj: + # NOTE(dulek): We ignore pods without annotation, those + # probably are hostNetworking. + continue + except Exception: + # TODO(dulek): We might want to print this exception. + malformed_count += 1 + continue + + if obj.obj_name() != objects.vif.PodState.obj_name(): + old_count += 1 + + if malformed_count == 0 and old_count == 0: + return UpgradeCheckResult(0, 'All annotations are updated.') + elif malformed_count > 0 and old_count == 0: + msg = ('You have %d malformed Kuryr pod annotations in your ' + 'deployment. This is not blocking the upgrade, but ' + 'consider investigating it.' % malformed_count) + return UpgradeCheckResult(1, msg) + elif old_count > 0: + msg = ('You have %d Kuryr pod annotations in old format. You need ' + 'to run `kuryr-status upgrade update-annotations` before ' + 'proceeding with the upgrade.' % old_count) + return UpgradeCheckResult(2, msg) + + def upgrade_check(self): + check_results = [] + + t = prettytable.PrettyTable(['Upgrade Check Results'], + hrules=prettytable.ALL) + t.align = 'l' + + for name, method in self.check_methods.items(): + result = method() + check_results.append(result) + cell = ( + 'Check: %(name)s\n' + 'Result: %(result)s\n' + 'Details: %(details)s' % + { + 'name': name, + 'result': UPGRADE_CHECK_MSG_MAP[result.code], + 'details': result.get_details(), + } + ) + t.add_row([cell]) + print(t) + + return max(res.code for res in check_results) + + def _convert_annotations(self, test_fn, update_fn): + updated_count = 0 + not_updated_count = 0 + malformed_count = 0 + pods = self.k8s.get('/api/v1/pods')['items'] + for pod in pods: + try: + obj = self._get_annotation(pod) + if not obj: + # NOTE(dulek): We ignore pods without annotation, those + # probably are hostNetworking. + continue + except Exception: + malformed_count += 1 + continue + + if test_fn(obj): + obj = update_fn(obj) + serialized = obj.obj_to_primitive() + try: + ann = { + constants.K8S_ANNOTATION_VIF: + jsonutils.dumps(serialized) + } + self.k8s.annotate( + pod['metadata']['selfLink'], ann, + pod['metadata']['resourceVersion']) + except exceptions.K8sClientException: + print('Error when updating annotation for pod %s/%s' % + (pod['metadata']['namespace'], + pod['metadata']['name'])) + not_updated_count += 1 + + updated_count += 1 + + t = prettytable.PrettyTable(['Stat', 'Number'], + hrules=prettytable.ALL) + t.align = 'l' + + cells = [['Updated annotations', updated_count], + ['Malformed annotations', malformed_count], + ['Annotations left', not_updated_count]] + for cell in cells: + t.add_row(cell) + print(t) + + def update_annotations(self): + def test_fn(obj): + return obj.obj_name() != objects.vif.PodState.obj_name() + + def update_fn(obj): + return vif.PodState(default_vif=obj) + + self._convert_annotations(test_fn, update_fn) + + def downgrade_annotations(self): + def test_fn(obj): + return obj.obj_name() == objects.vif.PodState.obj_name() + + def update_fn(obj): + return obj.default_vif + + self._convert_annotations(test_fn, update_fn) + + +def print_version(): + print(version.version_info.version_string()) + + +def add_parsers(subparsers): + upgrade_cmds = UpgradeCommands() + + upgrade = subparsers.add_parser( + 'upgrade', help='Actions related to upgrades between releases.') + sub = upgrade.add_subparsers() + + check = sub.add_parser('check', help='Check if upgrading is possible.') + check.set_defaults(action_fn=upgrade_cmds.upgrade_check) + + ann_update = sub.add_parser( + 'update-annotations', + help='Update annotations in K8s API to newest version.') + ann_update.set_defaults(action_fn=upgrade_cmds.update_annotations) + + ann_downgrade = sub.add_parser( + 'downgrade-annotations', + help='Downgrade annotations in K8s API to previous version (useful ' + 'when reverting a failed upgrade).') + ann_downgrade.set_defaults(action_fn=upgrade_cmds.downgrade_annotations) + + version_action = subparsers.add_parser('version') + version_action.set_defaults(action_fn=print_version) + + +def main(): + opt = cfg.SubCommandOpt( + 'category', title='command', + description='kuryr-status command or category to execute', + handler=add_parsers) + + conf = cfg.ConfigOpts() + conf.register_cli_opt(opt) + conf(sys.argv[1:]) + + os_vif.initialize() + objects.register_locally_defined_vifs() + + try: + return conf.category.action_fn() + except Exception: + print('Error:\n%s' % traceback.format_exc()) + # This is 255 so it's not confused with the upgrade check exit codes. + return 255 + + +if __name__ == '__main__': + main() diff --git a/kuryr_kubernetes/tests/unit/cmd/test_status.py b/kuryr_kubernetes/tests/unit/cmd/test_status.py new file mode 100644 index 000000000..d8e1d66c0 --- /dev/null +++ b/kuryr_kubernetes/tests/unit/cmd/test_status.py @@ -0,0 +1,183 @@ +# Copyright 2018 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslo_serialization import jsonutils +import six + +from kuryr_kubernetes.cmd import status +from kuryr_kubernetes import constants +from kuryr_kubernetes.objects import vif +from kuryr_kubernetes.tests import base as test_base + + +class TestStatusCmd(test_base.TestCase): + def setUp(self): + super(TestStatusCmd, self).setUp() + self.cmd = status.UpgradeCommands() + + def test_upgrade_result_get_details(self): + res = status.UpgradeCheckResult(0, 'a ' * 50) + + self.assertEqual( + (('a ' * 30).rstrip() + '\n' + (' ' * 9) + ('a ' * 20)).rstrip(), + res.get_details()) + + def test__get_annotation_missing(self): + pod = { + 'metadata': { + 'annotations': {} + } + } + + self.assertIsNone(self.cmd._get_annotation(pod)) + + def test__get_annotation_existing(self): + mock_obj = vif.PodState( + default_vif=vif.VIFMacvlanNested(vif_name='foo')) + + pod = { + 'metadata': { + 'annotations': { + constants.K8S_ANNOTATION_VIF: jsonutils.dumps( + mock_obj.obj_to_primitive()) + } + } + } + + obj = self.cmd._get_annotation(pod) + self.assertEqual(mock_obj, obj) + + @mock.patch('sys.stdout', new_callable=six.StringIO) + def _test_upgrade_check(self, code, code_name, m_stdout): + method_success_m = mock.Mock() + method_success_m.return_value = status.UpgradeCheckResult(0, 'foo') + method_code_m = mock.Mock() + method_code_m.return_value = status.UpgradeCheckResult(code, 'bar') + + self.cmd.check_methods = {'baz': method_success_m, + 'blah': method_code_m} + self.assertEqual(code, self.cmd.upgrade_check()) + + output = m_stdout.getvalue() + self.assertIn('baz', output) + self.assertIn('bar', output) + self.assertIn('foo', output) + self.assertIn('blah', output) + self.assertIn('Success', output) + self.assertIn(code_name, output) + + def test_upgrade_check_success(self): + self._test_upgrade_check(0, 'Success') + + def test_upgrade_check_warning(self): + self._test_upgrade_check(1, 'Warning') + + def test_upgrade_check_failure(self): + self._test_upgrade_check(2, 'Failure') + + def _test__check_annotations(self, ann_objs, code): + pods = { + 'items': [ + { + 'metadata': { + 'annotations': { + constants.K8S_ANNOTATION_VIF: ann + } + } + } for ann in ann_objs + ] + } + self.cmd.k8s = mock.Mock(get=mock.Mock(return_value=pods)) + res = self.cmd._check_annotations() + self.assertEqual(code, res.code) + + def test__check_annotations_succeed(self): + ann_objs = [ + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')), + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='bar')), + ] + ann_objs = [jsonutils.dumps(ann.obj_to_primitive()) + for ann in ann_objs] + + self._test__check_annotations(ann_objs, 0) + + def test__check_annotations_failure(self): + ann_objs = [ + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')), + vif.VIFMacvlanNested(vif_name='bar'), + ] + ann_objs = [jsonutils.dumps(ann.obj_to_primitive()) + for ann in ann_objs] + + self._test__check_annotations(ann_objs, 2) + + def test__check_annotations_malformed_and_old(self): + ann_objs = [ + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')), + vif.VIFMacvlanNested(vif_name='bar'), + ] + ann_objs = [jsonutils.dumps(ann.obj_to_primitive()) + for ann in ann_objs] + ann_objs.append('{}') + + self._test__check_annotations(ann_objs, 2) + + def test__check_annotations_malformed(self): + ann_objs = [ + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')), + ] + ann_objs = [jsonutils.dumps(ann.obj_to_primitive()) + for ann in ann_objs] + ann_objs.append('{}') + + self._test__check_annotations(ann_objs, 1) + + def _test__convert_annotations(self, method, calls): + self.cmd.k8s.annotate = mock.Mock() + + ann_objs = [ + ('foo', + vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo'))), + ('bar', vif.VIFMacvlanNested(vif_name='bar')), + ] + ann_objs = [(name, jsonutils.dumps(ann.obj_to_primitive())) + for name, ann in ann_objs] + + pods = { + 'items': [ + { + 'metadata': { + 'annotations': { + constants.K8S_ANNOTATION_VIF: ann + }, + 'selfLink': name, + 'resourceVersion': 1, + } + } for name, ann in ann_objs + ] + } + self.cmd.k8s = mock.Mock(get=mock.Mock(return_value=pods)) + method() + for args in calls: + self.cmd.k8s.annotate.assert_any_call(*args) + + def test_update_annotations(self): + self._test__convert_annotations(self.cmd.update_annotations, + [('bar', mock.ANY, 1)]) + + def test_downgrade_annotations(self): + self._test__convert_annotations(self.cmd.downgrade_annotations, + [('foo', mock.ANY, 1)]) diff --git a/kuryr_kubernetes/version.py b/kuryr_kubernetes/version.py index 87dac82ba..89900c997 100644 --- a/kuryr_kubernetes/version.py +++ b/kuryr_kubernetes/version.py @@ -12,5 +12,4 @@ import pbr.version -version_info = pbr.version.VersionInfo( - 'kuryr_kubernetes') +version_info = pbr.version.VersionInfo('kuryr_kubernetes') diff --git a/releasenotes/notes/stein-upgrade-226c8e7b735701ee.yaml b/releasenotes/notes/stein-upgrade-226c8e7b735701ee.yaml new file mode 100644 index 000000000..67d6eb502 --- /dev/null +++ b/releasenotes/notes/stein-upgrade-226c8e7b735701ee.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Before upgrading to T (0.7.x) run ``kuryr-status upgrade check`` to check + if upgrade is possible. In case of negative result refer to + kuryr-kubernetes documentation for mitigation steps. diff --git a/requirements.txt b/requirements.txt index f7df836b0..9a55d205f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.service!=1.28.1,>=1.24.0 # Apache-2.0 oslo.utils>=3.33.0 # Apache-2.0 os-vif!=1.8.0,>=1.7.0 # Apache-2.0 +PrettyTable<0.8,>=0.7.2 # BSD pyroute2>=0.5.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2) retrying!=1.3.0,>=1.2.3 # Apache-2.0 six>=1.10.0 # MIT diff --git a/setup.cfg b/setup.cfg index 56fd653ca..df3b5f712 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start kuryr-daemon = kuryr_kubernetes.cmd.daemon:start kuryr-cni = kuryr_kubernetes.cmd.cni:run + kuryr-status = kuryr_kubernetes.cmd.status:main kuryr_kubernetes.vif_translators = ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs