Improve functional tests

add logging in address association waiter.
add timeout for snapshot creation
fail test if resource can't be cleanuped

Change-Id: I7d805e5b49c359599e26c6f44d68289beee928f7
This commit is contained in:
Andrey Pavlov 2016-02-09 13:11:32 +03:00
parent ea4cab81f7
commit ae0941588d
2 changed files with 15 additions and 4 deletions

17
base.py
View File

@ -364,7 +364,10 @@ class EC2TestCase(base.BaseTestCase):
(cls.friendly_function_call_str(function, *pos_args, (cls.friendly_function_call_str(function, *pos_args,
**kw_args), **kw_args),
str((tb[0], tb[1], tb[2])))) str((tb[0], tb[1], tb[2]))))
cls.cleanUpItem(function, pos_args, kw_args) res = cls.cleanUpItem(function, pos_args, kw_args)
if not res:
fail_count += 1
LOG.error('Failure in cleanup for: %s' % str(kw_args))
except BaseException: except BaseException:
fail_count += 1 fail_count += 1
LOG.exception('Failure in cleanup for: %s' % str(kw_args)) LOG.exception('Failure in cleanup for: %s' % str(kw_args))
@ -375,6 +378,7 @@ class EC2TestCase(base.BaseTestCase):
@classmethod @classmethod
def cleanUpItem(cls, function, pos_args, kw_args): def cleanUpItem(cls, function, pos_args, kw_args):
attempts_left = 10 attempts_left = 10
interval = 1
deleted = False deleted = False
while not deleted and attempts_left > 0: while not deleted and attempts_left > 0:
try: try:
@ -390,11 +394,13 @@ class EC2TestCase(base.BaseTestCase):
waiter().wait_delete(obj_id) waiter().wait_delete(obj_id)
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e:
LOG.exception('Exception occured in cleanup waiting') LOG.exception('Exception occured in cleanup waiting')
return False
except botocore.exceptions.ClientError as e: except botocore.exceptions.ClientError as e:
error_code = e.response['Error']['Code'] error_code = e.response['Error']['Code']
for err in cls._VALID_CLEANUP_ERRORS: for err in cls._VALID_CLEANUP_ERRORS:
if err in error_code: if err in error_code:
deleted = True deleted = True
break
else: else:
hook_res = False hook_res = False
key = (function.__name__, error_code) key = (function.__name__, error_code)
@ -405,10 +411,13 @@ class EC2TestCase(base.BaseTestCase):
hook_res = hook(obj_id) hook_res = hook(obj_id)
if not hook_res: if not hook_res:
LOG.error('Cleanup failed: %s', e, exc_info=True) LOG.error('Cleanup failed: %s', e, exc_info=True)
return return False
LOG.error('Retrying cleanup due to: %s', e) LOG.error('Retrying cleanup due to: %s', e)
time.sleep(1) time.sleep(interval)
attempts_left -= 1 attempts_left -= 1
interval += 1
return deleted
@classmethod @classmethod
def friendly_function_name_simple(cls, call_able): def friendly_function_name_simple(cls, call_able):
@ -477,6 +486,8 @@ class EC2TestCase(base.BaseTestCase):
data = cls.client.describe_addresses( data = cls.client.describe_addresses(
Filters=[{'Name': 'association-id', 'Values': [assoc_id]}]) Filters=[{'Name': 'association-id', 'Values': [assoc_id]}])
LOG.debug('Addresses: %s' % str(data.get('Addresses')))
if ('Addresses' in data and len(data['Addresses']) == 1 and if ('Addresses' in data and len(data['Addresses']) == 1 and
data['Addresses'][0].get('InstanceId')): data['Addresses'][0].get('InstanceId')):
return 'available' return 'available'

View File

@ -236,7 +236,7 @@ class SnapshotPagingTest(scenario_base.BaseScenarioTest):
waiter = base.EC2Waiter(_create_snapshot) waiter = base.EC2Waiter(_create_snapshot)
cls.ids = list() cls.ids = list()
while len(cls.ids) < cls.SNAPSHOTS_COUNT: while len(cls.ids) < cls.SNAPSHOTS_COUNT:
time.sleep(2) time.sleep(10)
data = waiter.wait_for_result() data = waiter.wait_for_result()
snapshot_id = data['SnapshotId'] snapshot_id = data['SnapshotId']
cls.addResourceCleanUpStatic(cls.client.delete_snapshot, cls.addResourceCleanUpStatic(cls.client.delete_snapshot,