Make test_server_cfn_init a scenario test

Converting test_server_cfn_init to a scenario test is long overdue
since it is more of a scenario than an exercising of the heat API.

The heat-slow job has gone non-voting because this test fails
~%50 of the time due to bug #1297560. The server boot log is now
logged regardless of success or failure to build up data to diagnose
the issue.

This also adds several convenience functions to the test base class.

Co-Authored-By: Steven Hardy <shardy@redhat.com>
Related-Bug: #1297560
Change-Id: I077aeaf2bf8b292699eb20c5a75c59df35645913
This commit is contained in:
Steve Baker 2014-05-05 13:34:19 +12:00 committed by Steven Hardy
parent fc137b5dd7
commit 22c1660072
4 changed files with 230 additions and 122 deletions

View File

@ -1,121 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import testtools
from tempest.api.orchestration import base
from tempest.common.utils import data_utils
from tempest.common.utils.linux import remote_client
from tempest import config
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest import test
CONF = config.CONF
LOG = logging.getLogger(__name__)
class ServerCfnInitTestJSON(base.BaseOrchestrationTest):
existing_keypair = CONF.orchestration.keypair_name is not None
@classmethod
@test.safe_setup
def setUpClass(cls):
super(ServerCfnInitTestJSON, cls).setUpClass()
if not CONF.orchestration.image_ref:
raise cls.skipException("No image available to test")
template = cls.load_template('cfn_init_signal')
stack_name = data_utils.rand_name('heat')
if CONF.orchestration.keypair_name:
keypair_name = CONF.orchestration.keypair_name
else:
cls.keypair = cls._create_keypair()
keypair_name = cls.keypair['name']
# create the stack
cls.stack_identifier = cls.create_stack(
stack_name,
template,
parameters={
'key_name': keypair_name,
'flavor': CONF.orchestration.instance_type,
'image': CONF.orchestration.image_ref,
'network': cls._get_default_network()['id'],
'timeout': CONF.orchestration.build_timeout
})
@test.attr(type='slow')
@testtools.skipIf(existing_keypair, 'Server ssh tests are disabled.')
def test_can_log_into_created_server(self):
sid = self.stack_identifier
rid = 'SmokeServer'
# wait for create to complete.
self.client.wait_for_stack_status(sid, 'CREATE_COMPLETE')
resp, body = self.client.get_resource(sid, rid)
self.assertEqual('CREATE_COMPLETE', body['resource_status'])
# fetch the IP address from servers client, since we can't get it
# from the stack until stack create is complete
resp, server = self.servers_client.get_server(
body['physical_resource_id'])
# Check that the user can authenticate with the generated password
linux_client = remote_client.RemoteClient(server, 'ec2-user',
pkey=self.keypair[
'private_key'])
linux_client.validate_authentication()
@test.attr(type='slow')
def test_all_resources_created(self):
sid = self.stack_identifier
self.client.wait_for_resource_status(
sid, 'WaitHandle', 'CREATE_COMPLETE')
self.client.wait_for_resource_status(
sid, 'SmokeSecurityGroup', 'CREATE_COMPLETE')
self.client.wait_for_resource_status(
sid, 'SmokeKeys', 'CREATE_COMPLETE')
self.client.wait_for_resource_status(
sid, 'CfnUser', 'CREATE_COMPLETE')
self.client.wait_for_resource_status(
sid, 'SmokeServer', 'CREATE_COMPLETE')
try:
self.client.wait_for_resource_status(
sid, 'WaitCondition', 'CREATE_COMPLETE')
except (exceptions.StackResourceBuildErrorException,
exceptions.TimeoutException) as e:
# attempt to log the server console to help with debugging
# the cause of the server not signalling the waitcondition
# to heat.
resp, body = self.client.get_resource(sid, 'SmokeServer')
server_id = body['physical_resource_id']
LOG.debug('Console output for %s', server_id)
resp, output = self.servers_client.get_console_output(
server_id, None)
LOG.debug(output)
raise e
# wait for create to complete.
self.client.wait_for_stack_status(sid, 'CREATE_COMPLETE')
# This is an assert of great significance, as it means the following
# has happened:
# - cfn-init read the provided metadata and wrote out a file
# - a user was created and credentials written to the server
# - a cfn-signal was built which was signed with provided credentials
# - the wait condition was fulfilled and the stack has changed state
wait_status = json.loads(
self.get_stack_output(sid, 'WaitConditionStatus'))
self.assertEqual('smoke test complete', wait_status['00000'])

View File

@ -16,9 +16,12 @@
import logging
import os
import re
import six
import subprocess
import time
from heatclient import exc as heat_exceptions
import netaddr
from neutronclient.common import exceptions as exc
from novaclient import exceptions as nova_exceptions
@ -32,6 +35,7 @@ from tempest.common.utils.linux import remote_client
from tempest import config
from tempest import exceptions
from tempest.openstack.common import log
from tempest.openstack.common import timeutils
import tempest.test
CONF = config.CONF
@ -1069,3 +1073,98 @@ class OrchestrationScenarioTest(OfficialClientTest):
for net in networks['networks']:
if net['name'] == CONF.compute.fixed_network_name:
return net
@staticmethod
def _stack_output(stack, output_key):
"""Return a stack output value for a given key."""
return next((o['output_value'] for o in stack.outputs
if o['output_key'] == output_key), None)
def _ping_ip_address(self, ip_address, should_succeed=True):
cmd = ['ping', '-c1', '-w1', ip_address]
def ping():
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.wait()
return (proc.returncode == 0) == should_succeed
return tempest.test.call_until_true(
ping, CONF.orchestration.build_timeout, 1)
def _wait_for_resource_status(self, stack_identifier, resource_name,
status, failure_pattern='^.*_FAILED$'):
"""Waits for a Resource to reach a given status."""
fail_regexp = re.compile(failure_pattern)
build_timeout = CONF.orchestration.build_timeout
build_interval = CONF.orchestration.build_interval
start = timeutils.utcnow()
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
res = self.client.resources.get(
stack_identifier, resource_name)
except heat_exceptions.HTTPNotFound:
# ignore this, as the resource may not have
# been created yet
pass
else:
if res.resource_status == status:
return
if fail_regexp.search(res.resource_status):
raise exceptions.StackResourceBuildErrorException(
resource_name=res.resource_name,
stack_identifier=stack_identifier,
resource_status=res.resource_status,
resource_status_reason=res.resource_status_reason)
time.sleep(build_interval)
message = ('Resource %s failed to reach %s status within '
'the required time (%s s).' %
(res.resource_name, status, build_timeout))
raise exceptions.TimeoutException(message)
def _wait_for_stack_status(self, stack_identifier, status,
failure_pattern='^.*_FAILED$'):
"""
Waits for a Stack to reach a given status.
Note this compares the full $action_$status, e.g
CREATE_COMPLETE, not just COMPLETE which is exposed
via the status property of Stack in heatclient
"""
fail_regexp = re.compile(failure_pattern)
build_timeout = CONF.orchestration.build_timeout
build_interval = CONF.orchestration.build_interval
start = timeutils.utcnow()
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
stack = self.client.stacks.get(stack_identifier)
except heat_exceptions.HTTPNotFound:
# ignore this, as the stackource may not have
# been created yet
pass
else:
if stack.stack_status == status:
return
if fail_regexp.search(stack.stack_status):
raise exceptions.StackBuildErrorException(
stack_identifier=stack_identifier,
stack_status=stack.stack_status,
stack_status_reason=stack.stack_status_reason)
time.sleep(build_interval)
message = ('Stack %s failed to reach %s status within '
'the required time (%s s).' %
(stack.stack_name, status, build_timeout))
raise exceptions.TimeoutException(message)
def _stack_delete(self, stack_identifier):
try:
self.client.stacks.delete(stack_identifier)
except heat_exceptions.HTTPNotFound:
pass

View File

@ -62,7 +62,7 @@ Resources:
#!/bin/bash -v
/opt/aws/bin/cfn-init
/opt/aws/bin/cfn-signal -e 0 --data "`cat /tmp/smoke-status`" \
"WaitHandle"
--id smoke_status "WaitHandle"
WaitHandle:
Type: AWS::CloudFormation::WaitConditionHandle
WaitCondition:

View File

@ -0,0 +1,130 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from tempest import config
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest.scenario import manager
from tempest import test
CONF = config.CONF
LOG = logging.getLogger(__name__)
class CfnInitScenarioTest(manager.OrchestrationScenarioTest):
def setUp(self):
super(CfnInitScenarioTest, self).setUp()
if not CONF.orchestration.image_ref:
raise self.skipException("No image available to test")
self.client = self.orchestration_client
self.template_name = 'cfn_init_signal.yaml'
def assign_keypair(self):
self.stack_name = self._stack_rand_name()
if CONF.orchestration.keypair_name:
self.keypair = None
self.keypair_name = CONF.orchestration.keypair_name
else:
self.keypair = self.create_keypair()
self.keypair_name = self.keypair.id
def launch_stack(self):
net = self._get_default_network()
self.parameters = {
'key_name': self.keypair_name,
'flavor': CONF.orchestration.instance_type,
'image': CONF.orchestration.image_ref,
'timeout': CONF.orchestration.build_timeout,
'network': net['id'],
}
# create the stack
self.template = self._load_template(__file__, self.template_name)
self.client.stacks.create(
stack_name=self.stack_name,
template=self.template,
parameters=self.parameters)
self.stack = self.client.stacks.get(self.stack_name)
self.stack_identifier = '%s/%s' % (self.stack_name, self.stack.id)
self.addCleanup(self._stack_delete, self.stack_identifier)
def check_stack(self):
sid = self.stack_identifier
self._wait_for_resource_status(
sid, 'WaitHandle', 'CREATE_COMPLETE')
self._wait_for_resource_status(
sid, 'SmokeSecurityGroup', 'CREATE_COMPLETE')
self._wait_for_resource_status(
sid, 'SmokeKeys', 'CREATE_COMPLETE')
self._wait_for_resource_status(
sid, 'CfnUser', 'CREATE_COMPLETE')
self._wait_for_resource_status(
sid, 'SmokeServer', 'CREATE_COMPLETE')
server_resource = self.client.resources.get(sid, 'SmokeServer')
server_id = server_resource.physical_resource_id
server = self.compute_client.servers.get(server_id)
server_ip = server.networks[CONF.compute.network_for_ssh][0]
if not self._ping_ip_address(server_ip):
self._log_console_output(servers=[server])
self.fail(
"Timed out waiting for %s to become reachable" % server_ip)
try:
self._wait_for_resource_status(
sid, 'WaitCondition', 'CREATE_COMPLETE')
except (exceptions.StackResourceBuildErrorException,
exceptions.TimeoutException) as e:
raise e
finally:
# attempt to log the server console regardless of WaitCondition
# going to complete. This allows successful and failed cloud-init
# logs to be compared
self._log_console_output(servers=[server])
self._wait_for_stack_status(sid, 'CREATE_COMPLETE')
stack = self.client.stacks.get(sid)
# This is an assert of great significance, as it means the following
# has happened:
# - cfn-init read the provided metadata and wrote out a file
# - a user was created and credentials written to the server
# - a cfn-signal was built which was signed with provided credentials
# - the wait condition was fulfilled and the stack has changed state
wait_status = json.loads(
self._stack_output(stack, 'WaitConditionStatus'))
self.assertEqual('smoke test complete', wait_status['smoke_status'])
if self.keypair:
# Check that the user can authenticate with the generated
# keypair
try:
linux_client = self.get_remote_client(
server_ip, username='ec2-user')
linux_client.validate_authentication()
except (exceptions.ServerUnreachable,
exceptions.SSHTimeout) as e:
self._log_console_output(servers=[server])
raise e
@test.attr(type='slow')
@test.services('orchestration', 'compute')
def test_server_cfn_init(self):
self.assign_keypair()
self.launch_stack()
self.check_stack()