Allow run_update_ansible_action run with Mistral or Ansible

We extend run_update_ansible_action
to allow running the Ansible playbooks with Mistral
or executing directly thru Ansible.

This is needed  in case we need to run
exceptionally a task depending on Mistral
but Mistral is broken. For example, retry
an upgrade operation after having Mistral broken.

Change-Id: I15511b4f36260292e0ea4100b15b8e65a701b38b
(cherry picked from commit 5249fbb262)

tripleoclient/constants.py

@@ -72,3 +72,5 @@ DEPRECATED_SERVICES = {"OS::TripleO::Services::OpenDaylightApi":
                        "OpenDaylight to another networking backend. We "
                        "recommend you understand other networking "
                        "alternatives such as OVS or OVN. "}
+
+DEFAULT_VALIDATIONS_BASEDIR = '/usr/share/openstack-tripleo-validations'

tripleoclient/tests/test_utils.py

@@ -16,11 +16,15 @@
 
 import argparse
 import datetime
+import logging
 import mock
 import os.path
+import subprocess
 import tempfile
 from uuid import uuid4
 
+import sys
+
 from unittest import TestCase
 import yaml
 
@@ -29,6 +33,313 @@ from tripleoclient import exceptions
 from tripleoclient import utils
 
 
+class TestRunAnsiblePlaybook(TestCase):
+    def setUp(self):
+        self.unlink_patch = mock.patch('os.unlink')
+        self.addCleanup(self.unlink_patch.stop)
+        self.unlink_patch.start()
+        self.mock_log = mock.Mock('logging.getLogger')
+        python_version = sys.version_info[0]
+        self.ansible_playbook_cmd = "ansible-playbook-%s" % (python_version)
+
+    @mock.patch('os.path.exists', return_value=False)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_no_playbook(self, mock_run, mock_exists):
+        self.assertRaises(RuntimeError,
+                          utils.run_ansible_playbook,
+                          self.mock_log,
+                          '/tmp',
+                          'non-existing.yaml',
+                          'localhost,'
+                          )
+        mock_exists.assert_called_once_with('/tmp/non-existing.yaml')
+        mock_run.assert_not_called()
+
+    @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg'))
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_subprocess_error(self, mock_run, mock_exists, mock_mkstemp):
+        mock_process = mock.Mock()
+        mock_process.returncode = 1
+        mock_process.stdout.read.side_effect = ["Error\n"]
+        mock_run.return_value = mock_process
+
+        env = os.environ.copy()
+        env['ANSIBLE_LIBRARY'] = \
+            ('/root/.ansible/plugins/modules:'
+             '/usr/share/ansible/plugins/modules:'
+             '/usr/share/openstack-tripleo-validations/library')
+        env['ANSIBLE_LOOKUP_PLUGINS'] = \
+            ('root/.ansible/plugins/lookup:'
+             '/usr/share/ansible/plugins/lookup:'
+             '/usr/share/openstack-tripleo-validations/lookup_plugins')
+        env['ANSIBLE_CALLBACK_PLUGINS'] = \
+            ('~/.ansible/plugins/callback:'
+             '/usr/share/ansible/plugins/callback:'
+             '/usr/share/openstack-tripleo-validations/callback_plugins')
+        env['ANSIBLE_ROLES_PATH'] = \
+            ('/root/.ansible/roles:'
+             '/usr/share/ansible/roles:'
+             '/etc/ansible/roles:'
+             '/usr/share/openstack-tripleo-validations/roles')
+        env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log'
+
+        self.assertRaises(RuntimeError,
+                          utils.run_ansible_playbook,
+                          self.mock_log,
+                          '/tmp',
+                          'existing.yaml',
+                          'localhost,'
+                          )
+        mock_run.assert_called_once_with(self.mock_log,
+                                         [self.ansible_playbook_cmd,
+                                          '-u', 'root',
+                                          '-i', 'localhost,', '-v',
+                                          '-c', 'smart',
+                                          '/tmp/existing.yaml'],
+                                         env=env, retcode_only=False)
+
+    @mock.patch('os.path.isabs')
+    @mock.patch('os.path.exists', return_value=False)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_non_existing_config(self, mock_run, mock_exists, mock_isabs):
+        self.assertRaises(RuntimeError,
+                          utils.run_ansible_playbook, self.mock_log,
+                          '/tmp', 'existing.yaml', 'localhost,',
+                          '/tmp/foo.cfg'
+                          )
+        mock_exists.assert_called_once_with('/tmp/foo.cfg')
+        mock_isabs.assert_called_once_with('/tmp/foo.cfg')
+        mock_run.assert_not_called()
+
+    @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg'))
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_run_success_default(self, mock_run, mock_exists, mock_mkstemp):
+        mock_process = mock.Mock()
+        mock_process.returncode = 0
+        mock_run.return_value = mock_process
+
+        retcode = utils.run_ansible_playbook(self.mock_log,
+                                             '/tmp',
+                                             'existing.yaml',
+                                             'localhost,')
+        self.assertEqual(retcode, 0)
+        mock_exists.assert_called_once_with('/tmp/existing.yaml')
+
+        env = os.environ.copy()
+        env['ANSIBLE_LIBRARY'] = \
+            ('/root/.ansible/plugins/modules:'
+             '/usr/share/ansible/plugins/modules:'
+             '/usr/share/openstack-tripleo-validations/library')
+        env['ANSIBLE_LOOKUP_PLUGINS'] = \
+            ('root/.ansible/plugins/lookup:'
+             '/usr/share/ansible/plugins/lookup:'
+             '/usr/share/openstack-tripleo-validations/lookup_plugins')
+        env['ANSIBLE_CALLBACK_PLUGINS'] = \
+            ('~/.ansible/plugins/callback:'
+             '/usr/share/ansible/plugins/callback:'
+             '/usr/share/openstack-tripleo-validations/callback_plugins')
+        env['ANSIBLE_ROLES_PATH'] = \
+            ('/root/.ansible/roles:'
+             '/usr/share/ansible/roles:'
+             '/etc/ansible/roles:'
+             '/usr/share/openstack-tripleo-validations/roles')
+        env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log'
+
+        mock_run.assert_called_once_with(self.mock_log,
+                                         [self.ansible_playbook_cmd,
+                                          '-u', 'root',
+                                          '-i', 'localhost,', '-v',
+                                          '-c', 'smart',
+                                          '/tmp/existing.yaml'],
+                                         env=env, retcode_only=False)
+
+    @mock.patch('os.path.isabs')
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_run_success_ansible_cfg(self, mock_run, mock_exists, mock_isabs):
+        mock_process = mock.Mock()
+        mock_process.returncode = 0
+        mock_run.return_value = mock_process
+
+        retcode = utils.run_ansible_playbook(self.mock_log, '/tmp',
+                                             'existing.yaml', 'localhost,',
+                                             ansible_config='/tmp/foo.cfg')
+        self.assertEqual(retcode, 0)
+
+        mock_isabs.assert_called_once_with('/tmp/foo.cfg')
+
+        exist_calls = [mock.call('/tmp/foo.cfg'),
+                       mock.call('/tmp/existing.yaml')]
+        mock_exists.assert_has_calls(exist_calls, any_order=False)
+
+        env = os.environ.copy()
+        env['ANSIBLE_LIBRARY'] = \
+            ('/root/.ansible/plugins/modules:'
+             '/usr/share/ansible/plugins/modules:'
+             '/usr/share/openstack-tripleo-validations/library')
+        env['ANSIBLE_LOOKUP_PLUGINS'] = \
+            ('root/.ansible/plugins/lookup:'
+             '/usr/share/ansible/plugins/lookup:'
+             '/usr/share/openstack-tripleo-validations/lookup_plugins')
+        env['ANSIBLE_CALLBACK_PLUGINS'] = \
+            ('~/.ansible/plugins/callback:'
+             '/usr/share/ansible/plugins/callback:'
+             '/usr/share/openstack-tripleo-validations/callback_plugins')
+        env['ANSIBLE_ROLES_PATH'] = \
+            ('/root/.ansible/roles:'
+             '/usr/share/ansible/roles:'
+             '/etc/ansible/roles:'
+             '/usr/share/openstack-tripleo-validations/roles')
+        env['ANSIBLE_CONFIG'] = '/tmp/foo.cfg'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log'
+
+        mock_run.assert_called_once_with(self.mock_log,
+                                         [self.ansible_playbook_cmd,
+                                          '-u', 'root',
+                                          '-i', 'localhost,', '-v',
+                                          '-c', 'smart',
+                                          '/tmp/existing.yaml'],
+                                         env=env, retcode_only=False)
+
+    @mock.patch('tempfile.mkstemp', return_value=('foo', '/tmp/fooBar.cfg'))
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('tripleoclient.utils.run_command_and_log')
+    def test_run_success_connection_local(self, mock_run, mock_exists,
+                                          mok_mkstemp):
+        mock_process = mock.Mock()
+        mock_process.returncode = 0
+        mock_run.return_value = mock_process
+
+        retcode = utils.run_ansible_playbook(self.mock_log, '/tmp',
+                                             'existing.yaml',
+                                             'localhost,',
+                                             connection='local')
+        self.assertEqual(retcode, 0)
+        mock_exists.assert_called_once_with('/tmp/existing.yaml')
+        env = os.environ.copy()
+        env['ANSIBLE_LIBRARY'] = \
+            ('/root/.ansible/plugins/modules:'
+             '/usr/share/ansible/plugins/modules:'
+             '/usr/share/openstack-tripleo-validations/library')
+        env['ANSIBLE_LOOKUP_PLUGINS'] = \
+            ('root/.ansible/plugins/lookup:'
+             '/usr/share/ansible/plugins/lookup:'
+             '/usr/share/openstack-tripleo-validations/lookup_plugins')
+        env['ANSIBLE_CALLBACK_PLUGINS'] = \
+            ('~/.ansible/plugins/callback:'
+             '/usr/share/ansible/plugins/callback:'
+             '/usr/share/openstack-tripleo-validations/callback_plugins')
+        env['ANSIBLE_ROLES_PATH'] = \
+            ('/root/.ansible/roles:'
+             '/usr/share/ansible/roles:'
+             '/etc/ansible/roles:'
+             '/usr/share/openstack-tripleo-validations/roles')
+        env['ANSIBLE_CONFIG'] = '/tmp/fooBar.cfg'
+        env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        env['ANSIBLE_LOG_PATH'] = '/tmp/ansible.log'
+
+        mock_run.assert_called_once_with(self.mock_log,
+                                         [self.ansible_playbook_cmd,
+                                          '-u', 'root',
+                                          '-i', 'localhost,', '-v',
+                                          '-c', 'local',
+                                          '/tmp/existing.yaml'],
+                                         env=env, retcode_only=False)
+
+
+class TestRunCommandAndLog(TestCase):
+    def setUp(self):
+        self.mock_logger = mock.Mock(spec=logging.Logger)
+
+        self.mock_process = mock.Mock()
+        self.mock_process.stdout.readline.side_effect = ['foo\n', 'bar\n']
+        self.mock_process.wait.side_effect = [0]
+        self.mock_process.returncode = 0
+
+        mock_sub = mock.patch('subprocess.Popen',
+                              return_value=self.mock_process)
+        self.mock_popen = mock_sub.start()
+        self.addCleanup(mock_sub.stop)
+
+        self.cmd = ['exit', '0']
+        self.e_cmd = ['exit', '1']
+        self.log_calls = [mock.call('foo'),
+                          mock.call('bar')]
+
+    def test_success_default(self):
+        retcode = utils.run_command_and_log(self.mock_logger, self.cmd)
+        self.mock_popen.assert_called_once_with(self.cmd,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.STDOUT,
+                                                shell=False,
+                                                cwd=None, env=None)
+        self.assertEqual(retcode, 0)
+        self.mock_logger.warning.assert_has_calls(self.log_calls,
+                                                  any_order=False)
+
+    @mock.patch('subprocess.Popen')
+    def test_error_subprocess(self, mock_popen):
+        mock_process = mock.Mock()
+        mock_process.stdout.readline.side_effect = ['Error\n']
+        mock_process.wait.side_effect = [1]
+        mock_process.returncode = 1
+
+        mock_popen.return_value = mock_process
+
+        retcode = utils.run_command_and_log(self.mock_logger, self.e_cmd)
+        mock_popen.assert_called_once_with(self.e_cmd, stdout=subprocess.PIPE,
+                                           stderr=subprocess.STDOUT,
+                                           shell=False, cwd=None,
+                                           env=None)
+
+        self.assertEqual(retcode, 1)
+        self.mock_logger.warning.assert_called_once_with('Error')
+
+    def test_success_env(self):
+        test_env = os.environ.copy()
+        retcode = utils.run_command_and_log(self.mock_logger, self.cmd,
+                                            env=test_env)
+        self.mock_popen.assert_called_once_with(self.cmd,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.STDOUT,
+                                                shell=False,
+                                                cwd=None, env=test_env)
+        self.assertEqual(retcode, 0)
+        self.mock_logger.warning.assert_has_calls(self.log_calls,
+                                                  any_order=False)
+
+    def test_success_cwd(self):
+        test_cwd = '/usr/local/bin'
+        retcode = utils.run_command_and_log(self.mock_logger, self.cmd,
+                                            cwd=test_cwd)
+        self.mock_popen.assert_called_once_with(self.cmd,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.STDOUT,
+                                                shell=False,
+                                                cwd=test_cwd, env=None)
+        self.assertEqual(retcode, 0)
+        self.mock_logger.warning.assert_has_calls(self.log_calls,
+                                                  any_order=False)
+
+    def test_success_no_retcode(self):
+        run = utils.run_command_and_log(self.mock_logger, self.cmd,
+                                        retcode_only=False)
+        self.mock_popen.assert_called_once_with(self.cmd,
+                                                stdout=subprocess.PIPE,
+                                                stderr=subprocess.STDOUT,
+                                                shell=False,
+                                                cwd=None, env=None)
+        self.assertEqual(run, self.mock_process)
+        self.mock_logger.warning.assert_not_called()
+
+
 class TestWaitForStackUtil(TestCase):
     def setUp(self):
         self.mock_orchestration = mock.Mock()

tripleoclient/utils.py

@@ -26,6 +26,7 @@ import six
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 import yaml
 
@@ -36,10 +37,181 @@ from osc_lib.i18n import _
 from oslo_concurrency import processutils
 from six.moves import configparser
 
+from tripleo_common.utils import config
 from tripleoclient import constants
 from tripleoclient import exceptions
 
 
+LOG = logging.getLogger(__name__ + ".utils")
+
+
+def run_ansible_playbook(logger,
+                         workdir,
+                         playbook,
+                         inventory,
+                         ansible_config=None,
+                         retries=True,
+                         connection='smart',
+                         output_callback='json',
+                         python_interpreter=None,
+                         ssh_user='root',
+                         key=None,
+                         module_path=None,
+                         limit_hosts=None,
+                         tags='',
+                         skip_tags='',
+                         verbosity=1):
+    """Simple wrapper for ansible-playbook
+
+    :param logger: logger instance
+    :type logger: Logger
+
+    :param workdir: location of the playbook
+    :type workdir: String
+
+    :param playbook: playbook filename
+    :type playbook: String
+
+    :param inventory: either proper inventory file, or a coma-separated list
+    :type inventory: String
+
+    :param ansible_config: Pass either Absolute Path, or None to generate a
+    temporary file, or False to not manage configuration at all
+    :type ansible_config: String
+
+    :param retries: do you want to get a retry_file?
+    :type retries: Boolean
+
+    :param connection: connection type (local, smart, etc)
+    :type connection: String
+
+    :param output_callback: Callback for output format. Defaults to "json"
+    :type output_callback: String
+
+    :param python_interpreter: Absolute path for the Python interpreter
+    on the host where Ansible is run.
+    :type python_interpreter: String
+
+    :param ssh_user: user for the ssh connection
+    :type ssh_user: String
+
+    :param key: private key to use for the ssh connection
+    :type key: String
+
+    :param module_path: location of the ansible module and library
+    :type module_path: String
+
+    :param limit_hosts: limit the execution to the hosts
+    :type limit_hosts: String
+
+    :param tags: run specific tags
+    :type tags: String
+
+    :param skip_tags: skip specific tags
+    :type skip_tags: String
+
+    :param verbosity: verbosity level for Ansible execution
+    :type verbosity: Interger
+    """
+    env = os.environ.copy()
+
+    env['ANSIBLE_LIBRARY'] = \
+        ('/root/.ansible/plugins/modules:'
+         '/usr/share/ansible/plugins/modules:'
+         '%s/library' % constants.DEFAULT_VALIDATIONS_BASEDIR)
+    env['ANSIBLE_LOOKUP_PLUGINS'] = \
+        ('root/.ansible/plugins/lookup:'
+         '/usr/share/ansible/plugins/lookup:'
+         '%s/lookup_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR)
+    env['ANSIBLE_CALLBACK_PLUGINS'] = \
+        ('~/.ansible/plugins/callback:'
+         '/usr/share/ansible/plugins/callback:'
+         '%s/callback_plugins' % constants.DEFAULT_VALIDATIONS_BASEDIR)
+    env['ANSIBLE_ROLES_PATH'] = \
+        ('/root/.ansible/roles:'
+         '/usr/share/ansible/roles:'
+         '/etc/ansible/roles:'
+         '%s/roles' % constants.DEFAULT_VALIDATIONS_BASEDIR)
+    env['ANSIBLE_LOG_PATH'] = os.path.join(workdir, 'ansible.log')
+    env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+
+    cleanup = False
+    if ansible_config is None:
+        _, tmp_config = tempfile.mkstemp(prefix=playbook, suffix='ansible.cfg')
+        with open(tmp_config, 'w+') as f:
+            f.write("[defaults]\nstdout_callback = %s\n" % output_callback)
+            if not retries:
+                f.write("retry_files_enabled = False\n")
+            f.close()
+        env['ANSIBLE_CONFIG'] = tmp_config
+        cleanup = True
+
+    elif os.path.isabs(ansible_config):
+        if os.path.exists(ansible_config):
+            env['ANSIBLE_CONFIG'] = ansible_config
+        else:
+            raise RuntimeError('No such configuration file: %s' %
+                               ansible_config)
+    elif os.path.exists(os.path.join(workdir, ansible_config)):
+        env['ANSIBLE_CONFIG'] = os.path.join(workdir, ansible_config)
+
+    play = os.path.join(workdir, playbook)
+
+    if os.path.exists(play):
+        cmd = ["ansible-playbook-{}".format(sys.version_info[0]),
+               '-u', ssh_user,
+               '-i', inventory
+               ]
+
+        if 0 < verbosity < 6:
+            cmd.extend(['-' + ('v' * verbosity)])
+
+        if key is not None:
+            cmd.extend(['--private-key=%s' % key])
+
+        if module_path is not None:
+            cmd.extend(['--module-path=%s' % module_path])
+
+        if limit_hosts is not None:
+            cmd.extend(['-l %s' % limit_hosts])
+
+        if tags is not '':
+            cmd.extend(['-t %s' % tags])
+
+        if skip_tags is not '':
+            cmd.extend(['--skip_tags %s' % skip_tags])
+
+        if python_interpreter is not None:
+            cmd.extend(['-e', 'ansible_python_interpreter=%s' %
+                              python_interpreter])
+
+        cmd.extend(['-c', connection, play])
+
+        proc = run_command_and_log(logger, cmd, env=env, retcode_only=False)
+        proc.wait()
+        cleanup and os.unlink(tmp_config)
+        if proc.returncode != 0:
+            raise RuntimeError(proc.stdout.read())
+        return proc.returncode
+    else:
+        cleanup and os.unlink(tmp_config)
+        raise RuntimeError('No such playbook: %s' % play)
+
+
+def download_ansible_playbooks(client, stack_name, output_dir='/tmp'):
+
+    log = logging.getLogger(__name__ + ".download_ansible_playbooks")
+    stack_config = config.Config(client)
+    tmp_ansible_dir = tempfile.mkdtemp(prefix='tripleo-ansible-',
+                                       dir=output_dir)
+
+    log.warning(_('Downloading {0} ansible playbooks...').format(stack_name))
+    stack_config.write_config(stack_config.fetch_config(stack_name),
+                              stack_name,
+                              tmp_ansible_dir)
+    return tmp_ansible_dir
+
+
 def bracket_ipv6(address):
     """Put a bracket around address if it is valid IPv6
 
@@ -892,17 +1064,89 @@ def get_tripleo_ansible_inventory(inventory_file='',
             "Inventory file %s can not be found." % inventory_file)
 
 
-def run_update_ansible_action(log, clients, nodes, inventory, playbook,
-                              all_playbooks, action, ssh_user,
-                              skip_tags='', verbosity=1):
+def run_update_ansible_action(log, clients, nodes, inventory,
+                              playbook, all_playbooks, ssh_user,
+                              action=None, skip_tags='',
+                              verbosity='1', workdir='', priv_key=''):
+
     playbooks = [playbook]
     if playbook == "all":
         playbooks = all_playbooks
     for book in playbooks:
         log.debug("Running ansible playbook %s " % book)
-        action.update_ansible(clients, nodes=nodes, inventory_file=inventory,
-                              playbook=book, node_user=ssh_user,
-                              skip_tags=skip_tags, verbosity=verbosity)
+        if action:
+            action.update_ansible(clients, nodes=nodes,
+                                  inventory_file=inventory,
+                                  playbook=book, node_user=ssh_user,
+                                  skip_tags=skip_tags,
+                                  verbosity=verbosity)
+        else:
+            run_ansible_playbook(logger=LOG,
+                                 workdir=workdir,
+                                 playbook=book,
+                                 inventory=inventory,
+                                 ssh_user=ssh_user,
+                                 key=ssh_private_key(workdir, priv_key),
+                                 module_path='/usr/share/ansible-modules',
+                                 limit_hosts=nodes,
+                                 skip_tags=skip_tags)
+
+
+def ssh_private_key(workdir, key):
+    if not key:
+        return None
+    if (isinstance(key, six.string_types) and
+            os.path.exists(key)):
+        return key
+
+    path = os.path.join(workdir, 'ssh_private_key')
+    with open(path, 'w') as ssh_key:
+        ssh_key.write(key)
+    os.chmod(path, 0o600)
+    return path
+
+
+def parse_extra_vars(extra_var_strings):
+    """Parses extra variables like Ansible would.
+
+    Each element in extra_var_strings is like the raw value of -e
+    parameter of ansible-playbook command. It can either be very
+    simple 'key=val key2=val2' format or it can be '{ ... }'
+    representing a YAML/JSON object.
+
+    The 'key=val key2=val2' format gets processed as if it was
+    '{"key": "val", "key2": "val2"}' object, and all YAML/JSON objects
+    get shallow-merged together in the order as they appear in
+    extra_var_strings, latter objects taking precedence over earlier
+    ones.
+
+    :param extra_var_strings: unparsed value(s) of -e parameter(s)
+    :type extra_var_strings: list of strings
+
+    :returns dict representing a merged object of all extra vars
+    """
+    result = {}
+
+    for extra_var_string in extra_var_strings:
+        invalid_yaml = False
+
+        try:
+            parse_vars = yaml.safe_load(extra_var_string)
+        except yaml.YAMLError:
+            invalid_yaml = True
+
+        if invalid_yaml or not isinstance(parse_vars, dict):
+            try:
+                parse_vars = dict(
+                    item.split('=') for item in extra_var_string.split())
+            except ValueError:
+                raise ValueError(
+                    'Invalid format for {extra_var_string}'.format(
+                        extra_var_string=extra_var_string))
+
+        result.update(parse_vars)
+
+    return result
 
 
 def prepend_environment(environment_files, templates_dir, environment):
@@ -984,3 +1228,43 @@ def check_file_for_enabled_service(env_file):
 def check_deprecated_service_is_enabled(environment_files):
     for env_file in environment_files:
         check_file_for_enabled_service(env_file)
+
+
+def run_command_and_log(log, cmd, cwd=None, env=None, retcode_only=True):
+    """Run command and log output
+
+    :param log: logger instance for logging
+    :type log: Logger
+
+    :param cmd: command in list form
+    :type cmd: List
+
+    :param cwd: current worknig directory for execution
+    :type cmd: String
+
+    :param env: modified environment for command run
+    :type env: List
+
+    :param retcode_only: Returns only retcode instead or proc objec
+    :type retcdode_only: Boolean
+    """
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT, shell=False,
+                            cwd=cwd, env=env)
+    if retcode_only:
+        # TODO(aschultz): this should probably goto a log file
+        while True:
+            try:
+                line = proc.stdout.readline()
+            except StopIteration:
+                break
+            if line != b'':
+                if isinstance(line, bytes):
+                    line = line.decode('utf-8')
+                log.warning(line.rstrip())
+            else:
+                break
+        proc.stdout.close()
+        return proc.wait()
+    else:
+        return proc

tripleoclient/v1/overcloud_ffwd_upgrade.py

@@ -154,8 +154,8 @@ class FFWDUpgradeRun(command.Command):
         limit_hosts = ''
         oooutils.run_update_ansible_action(
             self.log, clients, limit_hosts, inventory,
-            constants.FFWD_UPGRADE_PLAYBOOK, [], package_update,
-            parsed_args.ssh_user, verbosity=verbosity)
+            constants.FFWD_UPGRADE_PLAYBOOK, [], parsed_args.ssh_user,
+            package_update, verbosity=verbosity)
 
 
 class FFWDUpgradeConverge(DeployOvercloud):

tripleoclient/v1/overcloud_update.py

@@ -162,8 +162,8 @@ class UpdateRun(command.Command):
         oooutils.run_update_ansible_action(self.log, clients, nodes, inventory,
                                            playbook,
                                            constants.MINOR_UPDATE_PLAYBOOKS,
-                                           package_update,
                                            parsed_args.ssh_user,
+                                           package_update,
                                            verbosity=verbosity)
 
 

tripleoclient/v1/overcloud_upgrade.py

@@ -211,10 +211,10 @@ class UpgradeRun(command.Command):
         oooutils.run_update_ansible_action(self.log, clients, limit_hosts,
                                            inventory, playbook,
                                            constants.MAJOR_UPGRADE_PLAYBOOKS,
-                                           package_update,
                                            parsed_args.ssh_user,
-                                           skip_tags=skip_tags,
-                                           verbosity=verbosity)
+                                           package_update,
+                                           skip_tags,
+                                           verbosity)
 
         playbooks = (constants.MAJOR_UPGRADE_PLAYBOOKS
                      if playbook == 'all' else playbook)