From 7c518a40d9bd2cc41e29b04c4e89d76f8eeae376 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Mon, 18 Jun 2018 09:08:15 -0700 Subject: [PATCH] Convert enable-ssh-admin.sh to python Instead of using the script from the templates, use python for the enable-ssh-admin logic. This will allow for more properly handling failures. This also fixes a race condition where sshd has not already started on some of the nodes before we try and connect via ssh. A timeout is added where we wait for the port to come up. If the timeout has passed and the port is still not up, then an exception is raised. Change-Id: I3431d2ec724a880baf0de8f586490d145bedf870 Closes-Bug: #1769230 (cherry picked from commit I3431d2ec724a880baf0de8f586490d145bedf870) --- tripleoclient/constants.py | 4 + .../overcloud_deploy/test_overcloud_deploy.py | 161 +++++++++++++++--- .../tests/workflows/test_deployment.py | 87 ++++++++++ tripleoclient/v1/overcloud_deploy.py | 8 +- tripleoclient/workflows/deployment.py | 137 ++++++++++++--- 5 files changed, 350 insertions(+), 47 deletions(-) create mode 100644 tripleoclient/tests/workflows/test_deployment.py diff --git a/tripleoclient/constants.py b/tripleoclient/constants.py index e198b691f..c6908a8e5 100644 --- a/tripleoclient/constants.py +++ b/tripleoclient/constants.py @@ -61,3 +61,7 @@ FFWD_UPGRADE_PREPARE_SCRIPT = ("#!/bin/bash \n" "rm -f /usr/libexec/os-apply-config/templates/" "etc/os-net-config/config.json || true \n") CEPH_UPGRADE_PREPARE_ENV = "environments/lifecycle/ceph-upgrade-prepare.yaml" + +ENABLE_SSH_ADMIN_TIMEOUT = 300 +ENABLE_SSH_ADMIN_STATUS_INTERVAL = 5 +ENABLE_SSH_ADMIN_SSH_PORT_TIMEOUT = 300 diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py index eaed97985..e580bc118 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py +++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py @@ -66,6 +66,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): os.unlink(self.parameter_defaults_env_file) self.cmd._download_missing_files_from_plan = self.real_download_missing + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.plan_management.tarball', @@ -98,7 +104,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_parameters_env, mock_breakpoints_cleanupm, mock_events, mock_tarball, - mock_get_horizon_url): + mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--ceph-storage-scale', '3'] verifylist = [ @@ -170,6 +180,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input.assert_called_with() + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.parameters.invoke_plan_env_workflows', @@ -208,7 +224,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_breakpoints_cleanup, mock_tarball, mock_postconfig, mock_get_overcloud_endpoint, mock_invoke_plan_env_wf, - mock_get_horizon_url): + mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--ceph-storage-scale', '3', '--control-flavor', 'oooq_control', '--no-cleanup'] @@ -302,6 +322,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertEqual(env_map.get('parameter_defaults'), parameters_env.get('parameter_defaults')) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.parameters.invoke_plan_env_workflows', @@ -333,14 +359,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): @mock.patch('shutil.copytree', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_tht_deploy_with_plan_environment_file( - self, mock_tmpdir, mock_copy, mock_time, mock_uuid1, - mock_get_template_contents, wait_for_stack_ready_mock, - mock_remove_known_hosts, mock_overcloudrc, mock_write_overcloudrc, - mock_create_tempest_deployer, mock_create_parameters_env, - mock_validate_args, mock_breakpoints_cleanup, - mock_tarball, mock_postconfig, - mock_get_overcloud_endpoint, mock_shutil_rmtree, - mock_invoke_plan_env_wf, mock_get_horizon_url): + self, mock_tmpdir, mock_copy, mock_time, mock_uuid1, + mock_get_template_contents, wait_for_stack_ready_mock, + mock_remove_known_hosts, mock_overcloudrc, mock_write_overcloudrc, + mock_create_tempest_deployer, mock_create_parameters_env, + mock_validate_args, + mock_breakpoints_cleanup, + mock_tarball, mock_postconfig, + mock_get_overcloud_endpoint, mock_shutil_rmtree, + mock_invoke_plan_env_wf, mock_get_horizon_url, + mock_list_plans, mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): + arglist = ['--templates', '-p', 'the-plan-environment.yaml'] verifylist = [ ('templates', '/usr/share/openstack-tripleo-heat-templates/'), @@ -437,6 +468,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): clients.tripleoclient.object_store.put_object.assert_called() self.assertTrue(mock_invoke_plan_env_wf.called) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.parameters.' @@ -476,7 +513,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_parameters_env, mock_validate_args, mock_breakpoints_cleanup, mock_tarball, mock_postconfig, mock_get_overcloud_endpoint, - mock_deprecated_params, mock_get_horizon_url): + mock_deprecated_params, mock_get_horizon_url, + mock_list_plans, mock_config_downlad, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--skip-deploy-identifier'] verifylist = [ @@ -533,6 +573,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): deploy_plan_call_input = deploy_plan_call[1]['workflow_input'] self.assertTrue(deploy_plan_call_input['skip_deploy_identifier']) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.plan_management.tarball', @@ -560,7 +606,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_deploy_postconfig, mock_breakpoints_cleanup, mock_events, mock_tarball, - mock_get_horizon_url): + mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '/home/stack/tripleo-heat-templates'] verifylist = [ @@ -644,6 +694,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.cmd.take_action, parsed_args) self.assertFalse(mock_deploy_tht.called) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.plan_management.tarball', @@ -663,7 +719,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_update_parameters, mock_post_config, mock_utils_endpoint, mock_utils_createrc, mock_utils_tempest, mock_tarball, - mock_get_horizon_url): + mock_get_horizon_url, mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager workflow_client = clients.workflow_engine @@ -873,6 +932,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): parsed_args) self.assertIn('/tmp/notthere', str(error)) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', @@ -886,7 +948,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_oc_endpoint, mock_create_ocrc, mock_create_tempest_deployer_input, - mock_get_horizon_url): + mock_get_horizon_url, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager workflow_client = clients.workflow_engine @@ -913,6 +978,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input.assert_called_with() + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.plan_management.tarball', @@ -945,7 +1016,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_deploy_postconfig, mock_breakpoints_cleanup, mock_events, mock_tarball, - mock_get_horizon_url): + mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--rhel-reg', '--reg-sat-url', 'https://example.com', @@ -1155,6 +1230,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertFalse(mock_create_ocrc.called) self.assertFalse(mock_create_tempest_deployer_input.called) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.plan_management.tarball', @@ -1173,7 +1254,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_oc_endpoint, mock_create_ocrc, mock_create_tempest_deployer_input, - mock_tarball, mock_get_horizon_url): + mock_tarball, mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager workflow_client = clients.workflow_engine @@ -1330,6 +1415,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.cmd.take_action, parsed_args) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') + @mock.patch( + 'tripleoclient.workflows.plan_management.list_deployment_plans', + autospec=True) @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' @@ -1367,7 +1458,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_breakpoints_cleanup, mock_tarball, mock_deploy_post_config, - mock_get_horizon_url): + mock_get_horizon_url, + mock_list_plans, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--ceph-storage-scale', '3', '--control-scale', '3', '--ntp-server', 'ntp'] @@ -1517,6 +1612,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertRaises(exceptions.StackInProgress, self.cmd.take_action, parsed_args) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', @@ -1532,7 +1630,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_overcloudrc, mock_write_overcloudrc, mock_overcloud_endpoint, mock_create_tempest_deployer_input, - mock_get_horizon_url): + mock_get_horizon_url, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager orchestration_client = clients.orchestration orchestration_client.stacks.get.return_value = mock.Mock() @@ -1547,6 +1648,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertNotCalled(self.cmd._predeploy_verify_capabilities) mock_create_tempest_deployer_input.assert_called_with() + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', @@ -1562,7 +1666,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_overcloudrc, mock_write_overcloudrc, mock_overcloud_endpoint, mock_create_tempest_deployer_input, - mock_get_horizon_url): + mock_get_horizon_url, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager orchestration_client = clients.orchestration orchestration_client.stacks.get.return_value = mock.Mock() @@ -1604,6 +1711,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertRaises(exceptions.InvalidConfiguration, self.cmd.take_action, parsed_args) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') + @mock.patch('tripleoclient.workflows.deployment.config_download') @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', @@ -1621,7 +1731,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_get_overcloud_endpoint, mock_workflows_bm, mock_provision, mock_tempest_deploy_input, - mock_get_horizon_url): + mock_get_horizon_url, + mock_config_download, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): arglist = ['--templates', '--deployed-server', '--disable-validations'] verifylist = [ ('templates', '/usr/share/openstack-tripleo-heat-templates/'), @@ -1661,6 +1774,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): parsed_args) self.assertFalse(mock_deploy_tmpdir.called) + @mock.patch('tripleoclient.workflows.deployment.get_overcloud_hosts') + @mock.patch('tripleoclient.workflows.deployment.enable_ssh_admin') @mock.patch('tripleoclient.workflows.deployment.get_horizon_url', autospec=True) @mock.patch('tripleoclient.workflows.deployment.config_download') @@ -1677,7 +1792,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_overcloudrc, mock_write_overcloudrc, mock_overcloud_endpoint, mock_create_tempest_deployer_input, - mock_config_download, mock_get_horizon_url): + mock_config_download, mock_get_horizon_url, + mock_enable_ssh_admin, + mock_get_overcloud_hosts): clients = self.app.client_manager orchestration_client = clients.orchestration orchestration_client.stacks.get.return_value = mock.Mock() @@ -1691,6 +1808,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.cmd.take_action(parsed_args) self.assertTrue(mock_deploy_tmpdir.called) + self.assertTrue(mock_enable_ssh_admin.called) + self.assertTrue(mock_get_overcloud_hosts.called) self.assertTrue(mock_config_download.called) def test_download_missing_files_from_plan(self): diff --git a/tripleoclient/tests/workflows/test_deployment.py b/tripleoclient/tests/workflows/test_deployment.py new file mode 100644 index 000000000..2db5d68b7 --- /dev/null +++ b/tripleoclient/tests/workflows/test_deployment.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from osc_lib.tests import utils + +from tripleoclient.workflows import deployment + + +class TestDeploymentWorkflows(utils.TestCommand): + + def setUp(self): + super(TestDeploymentWorkflows, self).setUp() + + self.app.client_manager.workflow_engine = self.workflow = mock.Mock() + self.tripleoclient = mock.Mock() + self.websocket = mock.Mock() + self.websocket.__enter__ = lambda s: self.websocket + self.websocket.__exit__ = lambda s, *exc: None + self.tripleoclient.messaging_websocket.return_value = self.websocket + self.app.client_manager.tripleoclient = self.tripleoclient + + self.message_success = iter([{ + "execution": {"id": "IDID"}, + "status": "SUCCESS", + "message": "Success.", + "registered_nodes": [], + }]) + self.message_failed = iter([{ + "execution": {"id": "IDID"}, + "status": "FAIL", + "message": "Fail.", + }]) + + @mock.patch('tripleoclient.workflows.deployment.wait_for_ssh_port') + @mock.patch('tripleoclient.workflows.deployment.time.sleep') + @mock.patch('tripleoclient.workflows.deployment.shutil.rmtree') + @mock.patch('tripleoclient.workflows.deployment.open') + @mock.patch('tripleoclient.workflows.deployment.tempfile') + @mock.patch('tripleoclient.workflows.deployment.subprocess.check_call') + def test_enable_ssh_admin(self, mock_check_call, mock_tempfile, + mock_open, mock_rmtree, mock_sleep, + mock_wait_for_ssh_port): + log = mock.Mock() + hosts = 'a', 'b', 'c' + ssh_user = 'test-user' + ssh_key = 'test-key' + + mock_tempfile.mkdtemp.return_value = '/foo' + mock_read = mock.Mock() + mock_read.read.return_value = 'key' + mock_open.return_value = mock_read + mock_state = mock.Mock() + mock_state.state = 'SUCCESS' + self.workflow.executions.get.return_value = mock_state + deployment.enable_ssh_admin(log, self.app.client_manager, + hosts, ssh_user, ssh_key) + + # once for ssh-keygen, then twice per host + self.assertEqual(7, mock_check_call.call_count) + + # execution ran + self.assertEqual(1, self.workflow.executions.create.call_count) + call_args = self.workflow.executions.create.call_args + self.assertEqual('tripleo.access.v1.enable_ssh_admin', call_args[0][0]) + self.assertEqual(('a', 'b', 'c'), + call_args[1]['workflow_input']['ssh_servers']) + self.assertEqual('test-user', + call_args[1]['workflow_input']['ssh_user']) + self.assertEqual('key', + call_args[1]['workflow_input']['ssh_private_key']) + + # tmpdir should be cleaned up + self.assertEqual(1, mock_rmtree.call_count) + self.assertEqual('/foo', mock_rmtree.call_args[0][0]) diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index 228050454..0f2545195 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -716,6 +716,8 @@ class DeployOvercloud(command.Command): ) parser.add_argument( '--overcloud-ssh-key', + default=os.path.join( + os.path.expanduser('~'), '.ssh', 'id_rsa'), help=_('Key path for ssh access to overcloud nodes.') ) parser.add_argument( @@ -979,9 +981,13 @@ class DeployOvercloud(command.Command): if parsed_args.config_download: print("Deploying overcloud configuration") + hosts = deployment.get_overcloud_hosts(self.clients, stack) + deployment.enable_ssh_admin(self.log, self.clients, + hosts, + parsed_args.overcloud_ssh_user, + parsed_args.overcloud_ssh_key) deployment.config_download(self.log, self.clients, stack, parsed_args.templates, - parsed_args.deployed_server, parsed_args.overcloud_ssh_user, parsed_args.overcloud_ssh_key, parsed_args.output_dir, diff --git a/tripleoclient/workflows/deployment.py b/tripleoclient/workflows/deployment.py index c2f595d52..abe026141 100644 --- a/tripleoclient/workflows/deployment.py +++ b/tripleoclient/workflows/deployment.py @@ -14,12 +14,16 @@ from __future__ import print_function import os import pprint import re +import shutil +import socket import subprocess +import tempfile import time from heatclient.common import event_utils from openstackclient import shell +from tripleoclient import constants from tripleoclient import exceptions from tripleoclient import utils @@ -96,46 +100,129 @@ def overcloudrc(workflow_client, **input_): **input_) -def config_download(log, clients, stack, templates, deployed_server, - ssh_user, ssh_key, output_dir, verbosity=1): +def get_overcloud_hosts(clients, stack): role_net_hostname_map = utils.get_role_net_hostname_map(stack) hostnames = [] for role in role_net_hostname_map: hostnames.extend(role_net_hostname_map[role].get('ctlplane', [])) - ips = [] + hosts = [] hosts_entry = utils.get_hosts_entry(stack) for hostname in hostnames: for line in hosts_entry.split('\n'): match = re.search('\s*%s\s*' % hostname, line) if match: - ips.append(line.split(' ')[0]) + hosts.append(line.split(' ')[0]) - script_path = os.path.join(templates, - 'deployed-server', - 'scripts', - 'enable-ssh-admin.sh') + return hosts - env = os.environ.copy() - env.update(dict(OVERCLOUD_HOSTS=' '.join(ips), - OVERCLOUD_SSH_USER=ssh_user)) - - if ssh_key: - env['OVERCLOUD_SSH_KEY'] = ssh_key - - proc = subprocess.Popen([script_path], env=env, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) +def wait_for_ssh_port(host): + start = int(time.time()) while True: - line = proc.stdout.readline().decode('utf-8') - if line: - log.info(line.rstrip()) - if line == '' and proc.poll() is not None: - break - if proc.returncode != 0: - raise RuntimeError('%s failed.' % script_path) + now = int(time.time()) + if (now - start) > constants.ENABLE_SSH_ADMIN_SSH_PORT_TIMEOUT: + raise exceptions.DeploymentError( + "Timed out waiting for port 22 from %s" % host) + try: + socket.socket().connect((host, 22)) + return + except socket.error: + pass + + time.sleep(1) + + +def enable_ssh_admin(log, clients, hosts, ssh_user, ssh_key): + print("Enabling ssh admin (tripleo-admin) for hosts:") + print(" ".join(hosts)) + print("Using ssh user %s for initial connection." % ssh_user) + print("Using ssh key at %s for initial connection." % ssh_key) + + ssh_options = ("-o ConnectionAttempts=6 " + "-o ConnectTimeout=30 " + "-o StrictHostKeyChecking=no " + "-o UserKnownHostsFile=/dev/null") + tmp_key_dir = tempfile.mkdtemp() + tmp_key_private = os.path.join(tmp_key_dir, 'id_rsa') + tmp_key_public = os.path.join(tmp_key_dir, 'id_rsa.pub') + tmp_key_comment = "TripleO split stack short term key" + + try: + tmp_key_command = ["ssh-keygen", "-N", "", "-t", "rsa", "-b", "4096", + "-f", tmp_key_private, "-C", tmp_key_comment] + subprocess.check_call(tmp_key_command, stderr=subprocess.STDOUT) + tmp_key_public_contents = open(tmp_key_public).read() + + for host in hosts: + wait_for_ssh_port(host) + copy_tmp_key_command = ["ssh"] + ssh_options.split() + copy_tmp_key_command += \ + ["-o", "StrictHostKeyChecking=no", + "-i", ssh_key, "-l", ssh_user, host, + "echo -e '\n%s' >> $HOME/.ssh/authorized_keys" % + tmp_key_public_contents] + print("Inserting TripleO short term key for %s" % host) + subprocess.check_call(copy_tmp_key_command, + stderr=subprocess.STDOUT) + + print("Starting ssh admin enablement workflow") + + workflow_client = clients.workflow_engine + + workflow_input = { + "ssh_user": ssh_user, + "ssh_servers": hosts, + "ssh_private_key": open(tmp_key_private).read(), + } + + execution = base.start_workflow( + workflow_client, + 'tripleo.access.v1.enable_ssh_admin', + workflow_input=workflow_input + ) + + start = int(time.time()) + while True: + now = int(time.time()) + if (now - start) > constants.ENABLE_SSH_ADMIN_TIMEOUT: + raise exceptions.DeploymentError( + "ssh admin enablement workflow - TIMED OUT.") + + time.sleep(1) + execution = workflow_client.executions.get(execution.id) + state = execution.state + + if state == 'RUNNING': + if (now - start) % constants.ENABLE_SSH_ADMIN_STATUS_INTERVAL\ + == 0: + print("ssh admin enablement workflow - RUNNING.") + continue + elif state == 'SUCCESS': + print("ssh admin enablement workflow - COMPLETE.") + break + elif state == 'FAILED': + raise exceptions.DeploymentError( + "ssh admin enablement workflow - FAILED.") + + for host in hosts: + rm_tmp_key_command = ["ssh"] + ssh_options.split() + rm_tmp_key_command += \ + ["-l", ssh_user, host, + "sed -i -e '/%s/d' $HOME/.ssh/authorized_keys" % + tmp_key_comment] + print("Removing TripleO short term key from %s" % host) + subprocess.check_call(rm_tmp_key_command, stderr=subprocess.STDOUT) + finally: + print("Removing short term keys locally") + shutil.rmtree(tmp_key_dir) + + print("Enabling ssh admin - COMPLETE.") + + +def config_download(log, clients, stack, templates, + ssh_user, ssh_key, output_dir, verbosity=1): workflow_client = clients.workflow_engine tripleoclients = clients.tripleoclient