Wire up a --skip-tags parameter for overcloud upgrade run cli

This exposes the --skip-tags which is useful for skipping the
service validations (--skip-tags validation). The expected
format is a comma-separated string for multiple values,
though 'validation' and 'pre-upgrade' are the only currently
supported tags for --skip-tags in the upgrade playbooks. The
full list of tags currently used in tripleo-heat-templates
upgrade_tasks is at [0]

  openstack overcloud upgrade run --nodes foo
                                  --skip-tags validation

[0]: 3eb0c62e47/tools/yaml-validate.py (L167)
Change-Id: Ie7fb8d9a388c6d53a31800406b03ddb8ed426401
Depends-On: I8544de64d3307e3dc925c1cecf2d9156e31d25a8
(cherry picked from commit d1c39c15c2)

releasenotes/notes/add-skip-tags-overcloud-upgrade-run-6aaf5925ffc02359.yaml

@@ -0,0 +1,16 @@
+---
+upgrade:
+  - |
+    This adds a --skip-tags parameter to the openstack overcloud upgrade run
+    command
+
+    .. code-block:: bash
+
+        openstack overcloud upgrade run --nodes compute-0 --skip-tags validation
+
+    This is useful for skipping those step 0 tasks (tagged "validation") that
+    check if services are running before allowing the upgrade to proceed, especially
+    if you must re-run the upgrade after a failed attempt and some services
+    cannot easily be started. The currently supported values for this are
+    validation and pre-upgrade, and they can be combined as "--skip-tags
+    'validation,pre-upgrade'" if required.

tripleoclient/constants.py

@@ -45,3 +45,4 @@ MINOR_UPDATE_PLAYBOOKS = ['update_steps_playbook.yaml',
 MAJOR_UPGRADE_PLAYBOOKS = ["upgrade_steps_playbook.yaml",
                            "deploy_steps_playbook.yaml",
                            "post_upgrade_steps_playbook.yaml"]
+MAJOR_UPGRADE_SKIP_TAGS = ['validation', 'pre-upgrade']

tripleoclient/tests/v1/overcloud_update/test_overcloud_update.py

@@ -140,7 +140,8 @@ class TestOvercloudUpdateRun(fakes.TestOvercloudUpdateRun):
                 nodes='Compute',
                 inventory_file=mock_open().read(),
                 playbook='fake-playbook.yaml',
-                ansible_queue_name=constants.UPDATE_QUEUE
+                ansible_queue_name=constants.UPDATE_QUEUE,
+                skip_tags=''
             )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -168,7 +169,8 @@ class TestOvercloudUpdateRun(fakes.TestOvercloudUpdateRun):
                     nodes='Compute',
                     inventory_file=mock_open().read(),
                     playbook=book,
-                    ansible_queue_name=constants.UPDATE_QUEUE
+                    ansible_queue_name=constants.UPDATE_QUEUE,
+                    skip_tags=''
                 )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -195,7 +197,8 @@ class TestOvercloudUpdateRun(fakes.TestOvercloudUpdateRun):
                     nodes=None,
                     inventory_file=mock_open().read(),
                     playbook=book,
-                    ansible_queue_name=constants.UPDATE_QUEUE
+                    ansible_queue_name=constants.UPDATE_QUEUE,
+                    skip_tags=''
                 )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',

tripleoclient/tests/v1/overcloud_upgrade/test_overcloud_upgrade.py

@@ -140,7 +140,8 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
                 nodes='Compute, Controller',
                 inventory_file=mock_open().read(),
                 playbook='fake-playbook.yaml',
-                ansible_queue_name=constants.UPGRADE_QUEUE
+                ansible_queue_name=constants.UPGRADE_QUEUE,
+                skip_tags=''
             )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -148,14 +149,16 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
     @mock.patch('os.path.expanduser')
     @mock.patch('oslo_concurrency.processutils.execute')
     @mock.patch('six.moves.builtins.open')
-    def test_upgrade_role_all_playbooks(
+    def test_upgrade_role_all_playbooks_skip_validation(
             self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
         mock_expanduser.return_value = '/home/fake/'
-        argslist = ['--roles', 'Compute', '--playbook', 'all']
+        argslist = ['--roles', 'Compute', '--playbook', 'all',
+                    '--skip-tags', 'validation']
         verifylist = [
             ('roles', 'Compute'),
             ('static_inventory', None),
-            ('playbook', 'all')
+            ('playbook', 'all'),
+            ('skip_tags', 'validation')
         ]
 
         parsed_args = self.check_parser(self.cmd, argslist, verifylist)
@@ -168,7 +171,8 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
                     nodes='Compute',
                     inventory_file=mock_open().read(),
                     playbook=book,
-                    ansible_queue_name=constants.UPGRADE_QUEUE
+                    ansible_queue_name=constants.UPGRADE_QUEUE,
+                    skip_tags='validation'
                 )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -176,15 +180,15 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
     @mock.patch('os.path.expanduser')
     @mock.patch('oslo_concurrency.processutils.execute')
     @mock.patch('six.moves.builtins.open')
-    def test_upgrade_nodes_with_playbook(
+    def test_upgrade_nodes_with_playbook_no_skip_tags(
             self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
         mock_expanduser.return_value = '/home/fake/'
         argslist = ['--nodes', 'compute-0, compute-1',
-                    '--playbook', 'fake-playbook.yaml']
+                    '--playbook', 'fake-playbook.yaml', ]
         verifylist = [
             ('nodes', 'compute-0, compute-1'),
             ('static_inventory', None),
-            ('playbook', 'fake-playbook.yaml')
+            ('playbook', 'fake-playbook.yaml'),
         ]
 
         parsed_args = self.check_parser(self.cmd, argslist, verifylist)
@@ -196,7 +200,8 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
                 nodes='compute-0, compute-1',
                 inventory_file=mock_open().read(),
                 playbook='fake-playbook.yaml',
-                ansible_queue_name=constants.UPGRADE_QUEUE
+                ansible_queue_name=constants.UPGRADE_QUEUE,
+                skip_tags=''
             )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -204,14 +209,14 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
     @mock.patch('os.path.expanduser')
     @mock.patch('oslo_concurrency.processutils.execute')
     @mock.patch('six.moves.builtins.open')
-    def test_upgrade_node_all_playbooks(
+    def test_upgrade_node_all_playbooks_skip_tags_default(
             self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
         mock_expanduser.return_value = '/home/fake/'
         argslist = ['--nodes', 'swift-1', '--playbook', 'all']
         verifylist = [
             ('nodes', 'swift-1'),
             ('static_inventory', None),
-            ('playbook', 'all')
+            ('playbook', 'all'),
         ]
 
         parsed_args = self.check_parser(self.cmd, argslist, verifylist)
@@ -224,7 +229,39 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
                     nodes='swift-1',
                     inventory_file=mock_open().read(),
                     playbook=book,
-                    ansible_queue_name=constants.UPGRADE_QUEUE
+                    ansible_queue_name=constants.UPGRADE_QUEUE,
+                    skip_tags=''
+                )
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    def test_upgrade_node_all_playbooks_skip_tags_all_supported(
+            self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'swift-1', '--playbook', 'all',
+                    '--skip-tags', 'pre-upgrade,validation']
+        verifylist = [
+            ('nodes', 'swift-1'),
+            ('static_inventory', None),
+            ('playbook', 'all'),
+            ('skip_tags', 'pre-upgrade,validation')
+        ]
+
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.cmd.take_action(parsed_args)
+            for book in constants.MAJOR_UPGRADE_PLAYBOOKS:
+                upgrade_ansible.assert_any_call(
+                    self.app.client_manager,
+                    nodes='swift-1',
+                    inventory_file=mock_open().read(),
+                    playbook=book,
+                    ansible_queue_name=constants.UPGRADE_QUEUE,
+                    skip_tags='pre-upgrade,validation'
                 )
 
     @mock.patch('tripleoclient.workflows.package_update.update_ansible',
@@ -250,10 +287,79 @@ class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
         mock_expanduser.return_value = '/home/fake/'
         argslist = ['--roles', 'Compute', '--nodes', 'overcloud-controller-1']
         verifylist = [
-            ('roles', ['Compute']),
+            ('roles', 'Compute'),
-            ('nodes', ['overcloud-controller-1']),
+            ('nodes', 'overcloud-controller-1'),
             ('static_inventory', None),
             ('playbook', 'all')
         ]
         self.assertRaises(ParserException, lambda: self.check_parser(
             self.cmd, argslist, verifylist))
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    # it is 'validation' not 'validations'
+    def test_upgrade_skip_tags_validations(self, mock_open, mock_execute,
+                                           mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'overcloud-compute-1',
+                    '--skip-tags', 'validations']
+        verifylist = [
+            ('nodes', 'overcloud-compute-1'),
+            ('static_inventory', None),
+            ('playbook', 'all'),
+            ('skip_tags', 'validations'),
+        ]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.assertRaises(exceptions.InvalidConfiguration,
+                              lambda: self.cmd.take_action(parsed_args))
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    # should only support the constants.MAJOR_UPGRADE_SKIP_TAGS
+    def test_upgrade_skip_tags_unsupported_validation_anything_else(
+            self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'overcloud-compute-1',
+                    '--skip-tags', 'validation,anything-else']
+        verifylist = [
+            ('nodes', 'overcloud-compute-1'),
+            ('static_inventory', None),
+            ('playbook', 'all'),
+            ('skip_tags', 'validation,anything-else'),
+        ]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.assertRaises(exceptions.InvalidConfiguration,
+                              lambda: self.cmd.take_action(parsed_args))
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    # should only support the constants.MAJOR_UPGRADE_SKIP_TAGS
+    def test_upgrade_skip_tags_unsupported_pre_upgrade_anything_else(
+            self, mock_open, mock_execute, mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'overcloud-compute-1',
+                    '--skip-tags', 'pre-upgrade,anything-else']
+        verifylist = [
+            ('nodes', 'overcloud-compute-1'),
+            ('static_inventory', None),
+            ('playbook', 'all'),
+            ('skip_tags', 'pre-upgrade,anything-else'),
+        ]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.assertRaises(exceptions.InvalidConfiguration,
+                              lambda: self.cmd.take_action(parsed_args))

tripleoclient/utils.py

@@ -823,11 +823,12 @@ def get_tripleo_ansible_inventory(inventory_file=''):
 
 
 def run_update_ansible_action(log, clients, nodes, inventory, playbook,
-                              queue, all_playbooks, action):
+                              queue, all_playbooks, action, skip_tags=''):
     playbooks = [playbook]
     if playbook == "all":
         playbooks = all_playbooks
     for book in playbooks:
         log.debug("Running ansible playbook %s " % book)
         action.update_ansible(clients, nodes=nodes, inventory_file=inventory,
-                              playbook=book, ansible_queue_name=queue)
+                              playbook=book, ansible_queue_name=queue,
+                              skip_tags=skip_tags)

tripleoclient/v1/overcloud_upgrade.py

@@ -20,6 +20,7 @@ from osc_lib.i18n import _
 
 from tripleoclient import command
 from tripleoclient import constants
+from tripleoclient import exceptions
 from tripleoclient import utils as oooutils
 from tripleoclient.v1.overcloud_deploy import DeployOvercloud
 from tripleoclient.workflows import package_update
@@ -152,8 +153,32 @@ class UpgradeRun(command.Command):
                                    'generated in '
                                    '~/tripleo-ansible-inventory.yaml')
                             )
+        parser.add_argument('--skip-tags',
+                            dest='skip_tags',
+                            action="store",
+                            default="",
+                            help=_('A string specifying the tag or comma '
+                                   'separated list of tags to be passed '
+                                   'as --skip-tags to ansible-playbook. '
+                                   'The currently supported values are '
+                                   '\'validation\' and \'pre-upgrade\'. '
+                                   'In particular \'validation\' is useful '
+                                   'if you must re-run following a failed '
+                                   'upgrade and some services cannot be '
+                                   'started. ')
+                            )
         return parser
 
+    def _validate_skip_tags(self, skip_tags):
+        tags_list = skip_tags.split(',')
+        for tag in tags_list:
+            tag = tag.strip()
+            if tag and tag not in constants.MAJOR_UPGRADE_SKIP_TAGS:
+                raise exceptions.InvalidConfiguration(
+                    "Unexpected tag %s. Supported values are %s" % (
+                        tag, constants.MAJOR_UPGRADE_SKIP_TAGS))
+        return skip_tags
+
     def take_action(self, parsed_args):
         self.log.debug("take_action(%s)" % parsed_args)
         clients = self.app.client_manager
@@ -164,11 +189,12 @@ class UpgradeRun(command.Command):
         playbook = parsed_args.playbook
         inventory = oooutils.get_tripleo_ansible_inventory(
             parsed_args.static_inventory)
+        skip_tags = self._validate_skip_tags(parsed_args.skip_tags)
         oooutils.run_update_ansible_action(self.log, clients, limit_hosts,
                                            inventory, playbook,
                                            constants.UPGRADE_QUEUE,
                                            constants.MAJOR_UPGRADE_PLAYBOOKS,
-                                           package_update)
+                                           package_update, skip_tags)
 
 
 class UpgradeConvergeOvercloud(DeployOvercloud):