Track fake stack updates for standalone/UC deploy

Puppet relies on stack_action UPDATE for some cases.
Track the ephemeral heat stacks fake state for tripleo
undercloud/standalone deployments to match the puppet
expectations. For such deployments, the heat stack
state is a fake (virtual), as we never
update but always create a new ephemeral heat stack.

When the deployment is finished w/o errors, create the mark
file (unique to the stack name) that is used to indicate to
puppet the stack_action has been changed from CREATE to
UPDATE. The indication is done via a drop-in file with
top level override containing either 'StackAction: CREATE'
or 'UPDATE'. The drop-in is created in the working/templates
directory and named <Stack_name>-stack-vstate-dropin.yaml.

When the deployment fails, remove the mark file, so
the serial re-deployments will be considered as creating
a stack. The --force-stack-update flag keeps the mark file
instead, so the serial re-deployment will be
considered (virtually) updating the heat stack.

For the --output-only mode, only warn users to control
the stack virtual state manually. The state is considered
CREATE, unless there is --force-stack-update specified.

For --dry-run, also log the expected stack virtual
state/action the deployment would go with.

Closes-bug: #1778505

Change-Id: I55dc83acb2ed5ee07b4cf57e25135e6201589ac4
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
This commit is contained in:
Bogdan Dobrelya 2018-06-26 13:38:17 +03:00
parent e70d027d53
commit 64436159e8
6 changed files with 225 additions and 50 deletions

View File

@ -20,6 +20,7 @@ OVERCLOUD_YAML_NAME = "overcloud.yaml"
OVERCLOUD_ROLES_FILE = "roles_data.yaml"
UNDERCLOUD_ROLES_FILE = "roles_data_undercloud.yaml"
UNDERCLOUD_OUTPUT_DIR = os.path.join(os.environ.get('HOME'))
STANDALONE_EPHEMERAL_STACK_VSTATE = '/var/lib/tripleo-heat-installer'
UNDERCLOUD_LOG_FILE = "install-undercloud.log"
UNDERCLOUD_CONF_PATH = os.path.join(UNDERCLOUD_OUTPUT_DIR, "undercloud.conf")
OVERCLOUD_NETWORKS_FILE = "network_data.yaml"

View File

@ -363,6 +363,53 @@ class TestDeployUndercloud(TestPluginV1):
env_files)
self.assertEqual(expected, results)
@mock.patch('yaml.safe_load', return_value={}, autospec=True)
@mock.patch('yaml.safe_dump', autospec=True)
@mock.patch('os.path.isfile', return_value=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_process_hieradata_overrides', autospec=True)
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_update_passwords_env', autospec=True)
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_normalize_user_templates', return_value=[], autospec=True)
@mock.patch('tripleoclient.utils.rel_or_abs_path', return_value={},
autospec=True)
@mock.patch('tripleoclient.utils.run_command_and_log', return_value=0,
autospec=True)
def test_setup_heat_environments_dropin(
self, mock_run, mock_paths, mock_norm, mock_update_pass_env,
mock_process_hiera, mock_open, mock_os, mock_yaml_dump,
mock_yaml_load):
parsed_args = self.check_parser(self.cmd,
['--local-ip', '127.0.0.1/8',
'--templates', 'tht_from',
'--output-dir', 'tht_to'], [])
dropin = 'tht_from/standalone-stack-vstate-dropin.yaml'
self.cmd.output_dir = 'tht_to'
self.cmd.tht_render = 'tht_from'
self.cmd.stack_action = 'UPDATE'
environment = self.cmd._setup_heat_environments(parsed_args)
self.assertIn(dropin, environment)
mock_open.assert_has_calls([mock.call(dropin, 'w')])
# unpack the dump yaml calls to verify if the produced stack update
# dropin matches our expectations
found_dropin = False
for call in mock_yaml_dump.call_args_list:
args, kwargs = call
for a in args:
if isinstance(a, mock.mock.MagicMock):
continue
if a.get('parameter_defaults', {}).get('StackAction', None):
self.assertTrue(
a['parameter_defaults']['StackAction'] == 'UPDATE')
found_dropin = True
self.assertTrue(found_dropin)
@mock.patch('os.path.isfile')
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@ -381,7 +428,7 @@ class TestDeployUndercloud(TestPluginV1):
def test_setup_heat_environments_default_plan_env(
self, mock_run, mock_update_pass_env, mock_process_hiera,
mock_process_multiple_environments, mock_hc_get_templ_cont,
mock_hc_process):
mock_hc_process, mock_os):
tmpdir = self.useFixture(fixtures.TempDir()).path
tht_from = os.path.join(tmpdir, 'tht-from')
@ -393,6 +440,7 @@ class TestDeployUndercloud(TestPluginV1):
self._setup_heat_environments(tmpdir, tht_from, plan_env_path,
mock_update_pass_env, mock_run)
@mock.patch('os.path.isfile')
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@ -411,7 +459,7 @@ class TestDeployUndercloud(TestPluginV1):
def test_setup_heat_environments_non_default_plan_env(
self, mock_run, mock_update_pass_env, mock_process_hiera,
mock_process_multiple_environments, mock_hc_get_templ_cont,
mock_hc_process):
mock_hc_process, mock_os):
tmpdir = self.useFixture(fixtures.TempDir()).path
tht_from = os.path.join(tmpdir, 'tht-from')
@ -484,12 +532,14 @@ class TestDeployUndercloud(TestPluginV1):
os.path.join(tht_render,
'tripleoclient-hosts-portmaps.yaml'),
'hiera_or.yaml',
os.path.join(tht_render, 'standalone-stack-vstate-dropin.yaml'),
os.path.join(tht_render, 'foo.yaml'),
os.path.join(tht_render, 'outside.yaml')]
environment = self.cmd._setup_heat_environments(parsed_args)
with mock.patch('os.path.isfile'):
environment = self.cmd._setup_heat_environments(parsed_args)
self.assertEqual(expected_env, environment)
self.assertEqual(expected_env, environment)
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_create_working_dirs', autospec=True)
@ -554,6 +604,8 @@ class TestDeployUndercloud(TestPluginV1):
env
)
@mock.patch('os.mkdir')
@mock.patch('six.moves.builtins.open')
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_populate_templates_dir')
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
@ -588,7 +640,8 @@ class TestDeployUndercloud(TestPluginV1):
mock_launchheat, mock_download, mock_tht,
mock_wait_for_port, mock_createdirs,
mock_cleanupdirs, mock_launchansible,
mock_tarball, mock_templates_dir):
mock_tarball, mock_templates_dir,
mock_open, mock_os):
parsed_args = self.check_parser(self.cmd,
['--local-ip', '127.0.0.1',

View File

@ -62,12 +62,13 @@ class TestUndercloudInstall(TestPluginV1):
@mock.patch('os.mkdir')
@mock.patch('tripleoclient.utils.write_env_file', autospec=True)
@mock.patch('subprocess.check_call', autospec=True)
def test_undercloud_install_with_heat_custom_output(self, mock_subprocess,
mock_wr,
mock_os, mock_copy):
def test_undercloud_install_with_heat_customized(self, mock_subprocess,
mock_wr,
mock_os, mock_copy):
self.conf.config(output_dir='/foo')
self.conf.config(templates='/usertht')
self.conf.config(roles_file='foo/roles.yaml')
arglist = ['--use-heat', '--no-validations']
arglist = ['--use-heat', '--no-validations', '--force-stack-update']
verifylist = []
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
@ -84,42 +85,31 @@ class TestUndercloudInstall(TestPluginV1):
'--standalone-role', 'Undercloud', '--stack', 'undercloud',
'--local-domain=localdomain',
'--local-ip=192.168.24.1/24',
'--templates=/usr/share/openstack-tripleo-heat-templates/',
'--templates=/usertht',
'--roles-file=foo/roles.yaml',
'--heat-native', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'docker.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'undercloud.yaml', '-e', '/home/stack/foo.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/ironic.yaml',
'-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/ironic-inspector.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/mistral.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/zaqar.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/tripleo-ui.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/tempest.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'public-tls-undercloud.yaml',
'/usertht/environments/docker.yaml', '-e',
'/usertht/environments/undercloud.yaml', '-e',
'/home/stack/foo.yaml', '-e',
'/usertht/environments/services/ironic.yaml', '-e',
'/usertht/environments/services/ironic-inspector.yaml', '-e',
'/usertht/environments/services/mistral.yaml', '-e',
'/usertht/environments/services/zaqar.yaml', '-e',
'/usertht/environments/services/tripleo-ui.yaml', '-e',
'/usertht/environments/services/tempest.yaml', '-e',
'/usertht/environments/public-tls-undercloud.yaml',
'--public-virtual-ip', '192.168.24.2',
'--control-virtual-ip', '192.168.24.3', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'ssl/tls-endpoints-public-ip.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'use-dns-for-vips.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-haproxy.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml', '--output-dir=/foo',
'--cleanup', '-e',
'/usertht/environments/ssl/tls-endpoints-public-ip.yaml', '-e',
'/usertht/environments/use-dns-for-vips.yaml', '-e',
'/usertht/environments/services/undercloud-haproxy.yaml', '-e',
'/usertht/environments/services/undercloud-keepalived.yaml',
'--output-dir=/foo', '--cleanup', '-e',
'/foo/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=install-undercloud.log'])
'--log-file=install-undercloud.log', '-e',
'/usertht/undercloud-stack-vstate-dropin.yaml',
'--force-stack-update'])
@mock.patch('shutil.copy')
@mock.patch('os.mkdir')
@ -280,7 +270,9 @@ class TestUndercloudInstall(TestPluginV1):
'--cleanup', '-e',
'/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=install-undercloud.log'])
'--log-file=install-undercloud.log', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])
@mock.patch('six.moves.builtins.open')
@mock.patch('shutil.copy')
@ -341,7 +333,9 @@ class TestUndercloudInstall(TestPluginV1):
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--debug', '--log-file=/foo/bar'])
'--debug', '--log-file=/foo/bar', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])
@mock.patch('shutil.copy')
@mock.patch('os.mkdir')
@ -401,7 +395,9 @@ class TestUndercloudInstall(TestPluginV1):
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=install-undercloud.log'])
'--log-file=install-undercloud.log', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])
class TestUndercloudUpgrade(TestPluginV1):
@ -494,7 +490,9 @@ class TestUndercloudUpgrade(TestPluginV1):
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=install-undercloud.log'])
'--log-file=install-undercloud.log', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])
@mock.patch('shutil.copy')
@mock.patch('os.mkdir')
@ -552,7 +550,9 @@ class TestUndercloudUpgrade(TestPluginV1):
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=install-undercloud.log'])
'--log-file=install-undercloud.log', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])
@mock.patch('shutil.copy')
@mock.patch('os.mkdir')
@ -613,4 +613,6 @@ class TestUndercloudUpgrade(TestPluginV1):
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--debug', '--log-file=install-undercloud.log'])
'--debug', '--log-file=install-undercloud.log', '-e',
'/usr/share/openstack-tripleo-heat-templates/'
'undercloud-stack-vstate-dropin.yaml'])

View File

@ -83,6 +83,8 @@ class Deploy(command.Command):
tmp_ansible_dir = None
roles_file = None
roles_data = None
stack_update_mark = None
stack_action = 'CREATE'
def _set_roles_file(self, file_name=None, templates_dir=None):
"""Set the roles file for the deployment
@ -160,6 +162,11 @@ class Deploy(command.Command):
raise exceptions.DeploymentError(msg)
return tar_filename
def _create_persistent_dirs(self):
"""Creates temporary working directories"""
if not os.path.exists(constants.STANDALONE_EPHEMERAL_STACK_VSTATE):
os.mkdir(constants.STANDALONE_EPHEMERAL_STACK_VSTATE)
def _create_working_dirs(self):
"""Creates temporary working directories"""
if self.output_dir and not os.path.exists(self.output_dir):
@ -563,6 +570,17 @@ class Deploy(command.Command):
parsed_args.hieradata_override,
parsed_args.standalone_role))
# Create a persistent drop-in file to indicate the stack
# virtual state changes
stack_vstate_dropin = os.path.join(self.tht_render,
'%s-stack-vstate-dropin.yaml' %
parsed_args.stack)
with open(stack_vstate_dropin, 'w') as dropin_file:
yaml.safe_dump(
{'parameter_defaults': {'StackAction': self.stack_action}},
dropin_file, default_flow_style=False)
environments.append(stack_vstate_dropin)
return environments + user_environments
def _prepare_container_images(self, env):
@ -686,8 +704,19 @@ class Deploy(command.Command):
parser.add_argument('-y', '--yes', default=False, action='store_true',
help=_("Skip yes/no prompt (assume yes)."))
parser.add_argument('--stack',
help=_("Stack name to create"),
help=_("Name for the ephemeral (one-time create "
"and forget) heat stack."),
default='standalone')
parser.add_argument('--force-stack-update',
dest='force_stack_update',
action='store_true',
default=False,
help=_("Do a virtual update of the ephemeral "
"heat stack (it cannot take real updates). "
"New or failed deployments "
"always have the stack_action=CREATE. This "
"option enforces stack_action=UPDATE."),
)
parser.add_argument('--output-dir',
dest='output_dir',
help=_("Directory to output state, processed heat "
@ -865,6 +894,9 @@ class Deploy(command.Command):
# prepare working spaces
self.output_dir = os.path.abspath(parsed_args.output_dir)
self._create_working_dirs()
# The state that needs to be persisted between serial deployments
# and cannot be contained in ephemeral heat stacks or working dirs
self._create_persistent_dirs()
# configure puppet
self._configure_puppet()
@ -878,6 +910,23 @@ class Deploy(command.Command):
rc = 1
try:
# NOTE(bogdando): Look for the unique virtual update mark matching
# the heat stack name we are going to create below. If found the
# mark, consider the stack action is UPDATE instead of CREATE.
mark_uuid = '_'.join(['update_mark', parsed_args.stack])
self.stack_update_mark = os.path.join(
constants.STANDALONE_EPHEMERAL_STACK_VSTATE,
mark_uuid)
# Prepare the heat stack action we want to start deployment with
if (os.path.isfile(self.stack_update_mark) or
parsed_args.force_stack_update):
self.stack_action = 'UPDATE'
self.log.warning(
_('The heat stack {0} action is {1}').format(
parsed_args.stack, self.stack_action))
# Launch heat.
orchestration_client = self._launch_heat(parsed_args)
# Wait for heat to be ready.
@ -886,6 +935,7 @@ class Deploy(command.Command):
stack_id = \
self._deploy_tripleo_heat_templates(orchestration_client,
parsed_args)
# Wait for complete..
status, msg = event_utils.poll_for_events(
orchestration_client, stack_id, nested_depth=6)
@ -920,15 +970,52 @@ class Deploy(command.Command):
tar_filename)
if not parsed_args.output_only and rc != 0:
# We only get here on error.
# Alter the stack virtual state for failed deployments
if (self.stack_update_mark and
not parsed_args.force_stack_update and
os.path.isfile(self.stack_update_mark)):
self.log.warning(
_('The heat stack %s virtual state/action is '
'is reset to CREATE. Use "--force-stack-update" to '
' set it forcefully to UPDATE') % parsed_args.stack)
self.log.warning(
_('Removing the stack virtual update mark file %s') %
self.stack_update_mark)
os.remove(self.stack_update_mark)
self.log.error(DEPLOY_FAILURE_MESSAGE.format(
self.heat_launch.install_tmp
))
raise exceptions.DeploymentError('Deployment failed.')
else:
# We only get here if no errors
self.log.warning(DEPLOY_COMPLETION_MESSAGE.format(
'~/undercloud-passwords.conf',
'~/stackrc'
))
if (self.stack_update_mark and
(not parsed_args.output_only or
parsed_args.force_stack_update)):
# Persist the unique mark file for this stack
# Do not update its atime file system attribute to keep its
# genuine timestamp for the 1st time the stack state had
# been (virtually) changed to match stack_action UPDATE
self.log.warning(
_('Writing the stack virtual update mark file %s') %
self.stack_update_mark)
open(self.stack_update_mark, 'wa').close()
elif parsed_args.output_only:
self.log.warning(
_('Not creating the stack %s virtual update mark file '
'in the --output-only mode! Re-run with '
'--force-stack-update, if you want to enforce it.') %
parsed_args.stack)
else:
self.log.warning(
_('Not creating the stack %s virtual update mark '
'file') % parsed_args.stack)
return rc
def take_action(self, parsed_args):

View File

@ -47,8 +47,18 @@ class InstallUndercloud(command.Command):
dest='use_heat',
action='store_true',
default=False,
help=_("Perform undercloud deploy using heat"),
help=_("Perform undercloud deploy using ephemeral (one-time "
"create and forget) heat stack and ansible."),
)
parser.add_argument('--force-stack-update',
dest='force_stack_update',
action='store_true',
default=False,
help=_("Do a virtual update of the ephemeral "
"heat stack. New or failed deployments "
"always have the stack_action=CREATE. This "
"option enforces stack_action=UPDATE."),
)
parser.add_argument(
'--no-validations',
dest='no_validations',
@ -81,7 +91,9 @@ class InstallUndercloud(command.Command):
cmd = undercloud_config.\
prepare_undercloud_deploy(
no_validations=no_validations,
verbose_level=self.app_args.verbose_level)
verbose_level=self.app_args.verbose_level,
force_stack_update=parsed_args.force_stack_update,
dry_run=parsed_args.dry_run)
else:
self.log.warning(_('Non-containerized undercloud deployment is '
'deprecated in Rocky cycle.'))
@ -114,7 +126,8 @@ class UpgradeUndercloud(InstallUndercloud):
yes=parsed_args.yes,
no_validations=parsed_args.
no_validations,
verbose_level=self.app_args.verbose_level)
verbose_level=self.app_args.verbose_level,
force_stack_update=parsed_args.force_stack_update)
self.log.warning("Running: %s" % ' '.join(cmd))
subprocess.check_call(cmd)
else:

View File

@ -240,7 +240,8 @@ def _generate_masquerade_networks():
def prepare_undercloud_deploy(upgrade=False, no_validations=False,
verbose_level=1, yes=False):
verbose_level=1, yes=False,
force_stack_update=False, dry_run=False):
"""Prepare Undercloud deploy command based on undercloud.conf"""
env_data = {}
@ -548,10 +549,28 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False,
deploy_args.append('--log-file=%s' % CONF['undercloud_log_file'])
# Always add a drop-in for the ephemeral undercloud heat stack
# virtual state tracking (the actual file will be created later)
stack_vstate_dropin = os.path.join(
CONF.get('templates') or constants.TRIPLEO_HEAT_TEMPLATES,
'undercloud-stack-vstate-dropin.yaml')
deploy_args += ["-e", stack_vstate_dropin]
if force_stack_update:
deploy_args += ["--force-stack-update"]
cmd = ["sudo", "openstack", "tripleo", "deploy", "--standalone",
"--standalone-role", "Undercloud", "--stack", "undercloud"]
cmd += deploy_args[:]
# In dry-run, also report the expected heat stack virtual state/action
if dry_run:
stack_update_mark = os.path.join(
constants.STANDALONE_EPHEMERAL_STACK_VSTATE,
'update_mark_undercloud')
if os.path.isfile(stack_update_mark) or force_stack_update:
LOG.warning(_('The heat stack undercloud virtual state/action '
' would be UPDATE'))
return cmd