5.0 Add retry for volume test in rados sys test
in system tests for ceph rados gw we call ostf as soon as deployment finish. And sometimes test on vilume creation fails with error status of volume. Second retry to run test is succesfull. It was caused by cinder scheduler, when we run ostf it do not see available spaces for volumes. So we add single run of volume test with retry and sleep in 60 second And if it passed - we runs full ostf suits Change-Id: I4e231438a2ffca853082e293dc099edabbc14a2a Closes-Bug: #1320248
This commit is contained in:
parent
d7f8696888
commit
578c5b0edb
|
@ -30,6 +30,7 @@ from fuelweb_test import logger
|
|||
from fuelweb_test.helpers.decorators import update_ostf
|
||||
from fuelweb_test.helpers.decorators import upload_manifests
|
||||
from fuelweb_test.models.nailgun_client import NailgunClient
|
||||
from fuelweb_test import ostf_test_mapping as map_ostf
|
||||
from fuelweb_test.settings import ATTEMPTS
|
||||
from fuelweb_test.settings import DEPLOYMENT_MODE_SIMPLE
|
||||
from fuelweb_test.settings import KVM_USE
|
||||
|
@ -37,6 +38,8 @@ from fuelweb_test.settings import NEUTRON
|
|||
from fuelweb_test.settings import NEUTRON_SEGMENT
|
||||
from fuelweb_test.settings import OPENSTACK_RELEASE
|
||||
from fuelweb_test.settings import OPENSTACK_RELEASE_UBUNTU
|
||||
from fuelweb_test.settings import OSTF_TEST_NAME
|
||||
from fuelweb_test.settings import OSTF_TEST_RETRIES_COUNT
|
||||
from fuelweb_test.settings import TIMEOUT
|
||||
|
||||
import fuelweb_test.settings as help_data
|
||||
|
@ -867,3 +870,38 @@ class FuelWebClient(object):
|
|||
node = self.get_nailgun_node_by_devops_node(node)
|
||||
assert_true(node['online'],
|
||||
'Node {0} is online'.format(node['mac']))
|
||||
|
||||
def run_ostf_repeatably(self, cluster_id, test_name=None,
|
||||
test_retries=None, checks=None):
|
||||
res = []
|
||||
passed_count = []
|
||||
failed_count = []
|
||||
test_nama_to_ran = test_name or OSTF_TEST_NAME
|
||||
retr = test_retries or OSTF_TEST_RETRIES_COUNT
|
||||
test_path = map_ostf.OSTF_TEST_MAPPING.get(test_nama_to_ran)
|
||||
logger.info('Test path is {0}'.format(test_path))
|
||||
|
||||
for i in range(0, retr):
|
||||
result = self.run_single_ostf_test(
|
||||
cluster_id=cluster_id, test_sets=['smoke', 'sanity'],
|
||||
test_name=test_path,
|
||||
retries=True)
|
||||
res.append(result)
|
||||
logger.info('res is {0}'.format(res))
|
||||
|
||||
logger.info('full res is {0}'.format(res))
|
||||
for element in res:
|
||||
[passed_count.append(test)
|
||||
for test in element if test.get(test_name) == 'success']
|
||||
[failed_count.append(test)
|
||||
for test in element if test.get(test_name) == 'failure']
|
||||
[failed_count.append(test)
|
||||
for test in element if test.get(test_name) == 'error']
|
||||
|
||||
if not checks:
|
||||
assert_true(
|
||||
len(passed_count) == test_retries,
|
||||
'not all retries were successful,'
|
||||
' fail {0} retries'.format(len(failed_count)))
|
||||
else:
|
||||
return failed_count
|
||||
|
|
|
@ -23,6 +23,7 @@ from fuelweb_test.helpers import os_actions
|
|||
from fuelweb_test.helpers import checkers
|
||||
from fuelweb_test.helpers.checkers import check_ceph_health
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_on_error
|
||||
from fuelweb_test import ostf_test_mapping as map_ostf
|
||||
from fuelweb_test import settings
|
||||
from fuelweb_test import logger
|
||||
from fuelweb_test.tests.base_test_case import SetupEnvironment
|
||||
|
@ -257,6 +258,25 @@ class CephRadosGW(TestBasic):
|
|||
remote = self.fuel_web.get_ssh_for_node('slave-01')
|
||||
check_ceph_health(remote)
|
||||
|
||||
def _check():
|
||||
# Run volume test several times with hope that it pass
|
||||
test_path = map_ostf.OSTF_TEST_MAPPING.get(
|
||||
'Create volume and attach it to instance')
|
||||
logger.debug('Start to run test {0}'.format(test_path))
|
||||
self.fuel_web.run_single_ostf_test(
|
||||
cluster_id, test_sets=['smoke'],
|
||||
test_name=test_path,
|
||||
should_fail=0)
|
||||
try:
|
||||
_check()
|
||||
except AssertionError:
|
||||
logger.debug(AssertionError)
|
||||
logger.debug("Test failed from first probe,"
|
||||
" we sleep 60 second try one more time "
|
||||
"and if it fails again - test will fails ")
|
||||
time.sleep(60)
|
||||
_check()
|
||||
|
||||
# Run ostf
|
||||
self.fuel_web.run_ostf(
|
||||
cluster_id=cluster_id,
|
||||
|
|
|
@ -12,49 +12,17 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from proboscis import asserts
|
||||
|
||||
from proboscis import test
|
||||
|
||||
from fuelweb_test.helpers.decorators import log_snapshot_on_error
|
||||
from fuelweb_test import ostf_test_mapping as map_ostf
|
||||
from fuelweb_test import settings as hlp_date
|
||||
from fuelweb_test.tests import base_test_case
|
||||
from fuelweb_test import logger
|
||||
|
||||
|
||||
@test(groups=["ostf_repeatable_tests"])
|
||||
class OstfRepeatableTests(base_test_case.TestBasic):
|
||||
|
||||
def _run_OSTF(self, cluster_id):
|
||||
res = []
|
||||
d_key = hlp_date.OSTF_TEST_NAME
|
||||
passed_count = []
|
||||
failed_count = []
|
||||
test_path = map_ostf.OSTF_TEST_MAPPING.get(hlp_date.OSTF_TEST_NAME)
|
||||
logger.info('Test path is {0}'.format(test_path))
|
||||
|
||||
for i in range(0, hlp_date.OSTF_TEST_RETRIES_COUNT):
|
||||
result = self.fuel_web.run_single_ostf_test(
|
||||
cluster_id=cluster_id, test_sets=['smoke', 'sanity'],
|
||||
test_name=test_path,
|
||||
retries=True)
|
||||
res.append(result)
|
||||
logger.info('res is {0}'.format(res))
|
||||
|
||||
logger.info('full res is {0}'.format(res))
|
||||
for element in res:
|
||||
[passed_count.append(test)
|
||||
for test in element if test.get(d_key) == 'success']
|
||||
[failed_count.append(test)
|
||||
for test in element if test.get(d_key) == 'failure']
|
||||
[failed_count.append(test)
|
||||
for test in element if test.get(d_key) == 'error']
|
||||
|
||||
asserts.assert_true(
|
||||
len(passed_count) == hlp_date.OSTF_TEST_RETRIES_COUNT,
|
||||
'not all retries were successful,'
|
||||
' fail {0} retries'.format(len(failed_count)))
|
||||
|
||||
@test(depends_on=[base_test_case.SetupEnvironment.prepare_slaves_3],
|
||||
groups=["create_delete_ip_n_times_nova_vlan"])
|
||||
@log_snapshot_on_error
|
||||
|
@ -92,7 +60,7 @@ class OstfRepeatableTests(base_test_case.TestBasic):
|
|||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self._run_OSTF(cluster_id)
|
||||
self.fuel_web.run_ostf_repeatably(cluster_id)
|
||||
|
||||
self.env.make_snapshot("create_delete_ip_n_times_nova_vlan")
|
||||
|
||||
|
@ -130,7 +98,7 @@ class OstfRepeatableTests(base_test_case.TestBasic):
|
|||
self.fuel_web.deploy_cluster_wait(cluster_id)
|
||||
|
||||
self.fuel_web.verify_network(cluster_id)
|
||||
self._run_OSTF(cluster_id)
|
||||
self.fuel_web.run_ostf_repeatably(cluster_id)
|
||||
|
||||
self.env.make_snapshot("create_delete_ip_n_times_nova_flat")
|
||||
|
||||
|
@ -139,4 +107,4 @@ class OstfRepeatableTests(base_test_case.TestBasic):
|
|||
def run_ostf_n_times_against_custom_deployment(self):
|
||||
cluster_id = self.fuel_web.client.get_cluster_id(
|
||||
hlp_date.DEPLOYMENT_NAME)
|
||||
self._run_OSTF(cluster_id)
|
||||
self.fuel_web.run_ostf_repeatably(cluster_id)
|
||||
|
|
Loading…
Reference in New Issue