openstack
/
fuel-qa
Code Issues Proposed changes

Create test plan when new swarm runner is started 

Modify reporter script, so it can parse results of
system tests that are running at the moment.
Also if test failed try to find link to Launchpad
bug in previous results for the same test case. Copy
link to bug only if it's targeted to given project
and milestone and its status isn't 'Fix Released'.
In case linked bug is marked as duplicate of another
bug, check the status of that bug. If linked bug is
private or doesn't exist skip it.

Change-Id: I2392c81bcad6273ecbabfee479b74263fc7c0fbf
Closes-bug: #1419865
This commit is contained in:
Artem Panchenko 2015-02-10 23:03:37 +02:00
parent 2ba17e5dda
commit 95891d69e0
6 changed files with 175 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fuelweb_test/requirements.txt
View File

@ -12,3 +12,4 @@ python-cinderclient>=1.0.5
python-neutronclient>=2.0
Jinja2
AllPairs==2.0.1
launchpadlib

3
fuelweb_test/testrail/builds.py
View File

@ -46,6 +46,9 @@ class Build():
if number == 'latest':
job_info = self.get_job_info(depth=0)
self.number = job_info["lastCompletedBuild"]["number"]
elif number == 'latest_started':
job_info = self.get_job_info(depth=0)
self.number = job_info["lastBuild"]["number"]
else:
self.number = int(number)

40
fuelweb_test/testrail/launchpad_client.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from launchpadlib.launchpad import Launchpad
class LaunchpadBug(object):
def __init__(self, bug_id):
self.launchpad = Launchpad.login_anonymously('just testing',
'production',
'.cache')
self.bug = self.launchpad.bugs[int(bug_id)]
@property
def targets(self):
return [{'project': str(task.bug_target_name).split('/')[0],
'milestone': str(task.milestone).split('/')[-1],
'status': str(task.status)} for task in self.bug_tasks]
def get_duplicate_of(self):
bug = self.bug
duplicates = []
while bug.duplicate_of and bug.id not in duplicates:
duplicates.append(bug.id)
bug = self.launchpad.load(str(bug.duplicate_of))
return LaunchpadBug(bug.id)
def __getattr__(self, item):
return self.bug.__getattr__(item)

109
fuelweb_test/testrail/report.py
View File

@ -22,6 +22,8 @@ from optparse import OptionParser
from builds import Build
from builds import get_jobs_for_view
from launchpad_client import LaunchpadBug
from settings import LaunchpadSettings
from settings import logger
from settings import TestRailSettings
from testrail_client import TestRailProject
@ -29,7 +31,7 @@ from testrail_client import TestRailProject
class TestResult():
def __init__(self, name, group, status, duration, url=None,
version=None, description=None):
version=None, description=None, launchpad_bug=None):
self.name = name
self.group = group
self._status = status
@ -37,6 +39,7 @@ class TestResult():
self.url = url
self.version = version
self.description = description
self.launchpad_bug = launchpad_bug
self.available_statuses = {
'passed': ['passed', 'fixed'],
'failed': ['failed', 'regression'],
@ -84,9 +87,9 @@ def retry(count=3):
return wrapped
def get_downstream_builds(jenkins_build_data):
return [{'name': b['jobName'], 'number': b['buildNumber']}
for b in jenkins_build_data['subBuilds']]
def get_downstream_builds(jenkins_build_data, status=None):
return [{'name': b['jobName'], 'number': b['buildNumber'],
'result': b['result']} for b in jenkins_build_data['subBuilds']]
def get_version(jenkins_build_data):
@ -120,9 +123,13 @@ def get_tests_results(systest_build):
return tests_results
def publish_results(project, test_plan, tests_suite, config_id, results):
def publish_results(project, milestone_id, test_plan,
tests_suite, config_id, results):
test_run_id = [run['id'] for run in test_plan['entries'][0]['runs'] if
config_id in run['config_ids']][0]
previous_tests_runs = project.get_previous_runs(milestone_id=milestone_id,
config_id=config_id)
cases = project.get_cases(suite=tests_suite)
tests = project.get_tests(run_id=test_run_id)
results_to_publish = []
@ -136,8 +143,18 @@ def publish_results(project, test_plan, tests_suite, config_id, results):
continue
existing_results_versions = [r['version'] for r in
project.get_results_for_test(test['id'])]
if result.version not in existing_results_versions:
results_to_publish.append(result)
if result.version in existing_results_versions:
continue
if result.status != 'passed':
run_ids = [run['id'] for run in previous_tests_runs]
case_id = project.get_case_by_group(suite=tests_suite,
group=result.group,
cases=cases)['id']
previous_results = project.get_all_results_for_case(
run_ids=run_ids,
case_id=case_id)
result.launchpad_bug = get_existing_bug_link(previous_results)
results_to_publish.append(result)
try:
if len(results_to_publish) > 0:
project.add_results_for_cases(run_id=test_run_id,
@ -151,6 +168,36 @@ def publish_results(project, test_plan, tests_suite, config_id, results):
return results_to_publish
@retry(count=3)
def get_existing_bug_link(previous_results):
results_with_bug = [result for result in previous_results if
result["custom_launchpad_bug"] is not None]
if not results_with_bug:
return
for result in sorted(results_with_bug,
key=lambda k: k['created_on'],
reverse=True):
try:
bug_id = int(result["custom_launchpad_bug"].strip('/').split(
'/')[-1])
except ValueError:
logger.warning('Link "{0}" doesn\'t contain bug id.'.format(
result["custom_launchpad_bug"]))
continue
try:
bug = LaunchpadBug(bug_id).get_duplicate_of()
except KeyError:
logger.error("Bug with id '{bug_id}' is private or "
"doesn't exist.".format(bug_id=bug_id))
return
for target in bug.targets:
if target['project'] == LaunchpadSettings.project and\
target['milestone'] == LaunchpadSettings.milestone and\
target['status'] not in LaunchpadSettings.closed_statuses:
return result["custom_launchpad_bug"]
def main():
parser = OptionParser(
@ -164,6 +211,8 @@ def main():
help='Jenkins swarm runner build number')
parser.add_option("-w", "--view", dest="jenkins_view", default=False,
help="Get system tests jobs from Jenkins view")
parser.add_option("-l", "--live", dest="live_report", action="store_true",
help="Get tests results from running swarm")
parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False,
help="Enable debug output")
@ -173,6 +222,9 @@ def main():
if options.verbose:
logger.setLevel(DEBUG)
if options.live_report:
options.build_number = 'latest_started'
tests_results_centos = []
tests_results_ubuntu = []
case_group = TestRailSettings.test_suite
@ -194,6 +246,11 @@ def main():
milestone, iso_number = get_version(runner_build.build_data)
for systest_build in tests_jobs:
if systest_build['result'] is None:
logger.debug("Skipping '{0}' job (build #{1}) because it's still "
"running...".format(systest_build['name'],
systest_build['number'],))
continue
if 'centos' in systest_build['name'].lower():
tests_results_centos.extend(get_tests_results(systest_build))
elif 'ubuntu' in systest_build['name'].lower():
@ -204,8 +261,12 @@ def main():
password=TestRailSettings.password,
project=TestRailSettings.project)
milestone = project.get_milestone(name=milestone)
test_plan_name = '{milestone} {case_group} iso #{iso_number}'.format(
milestone=milestone, case_group=case_group, iso_number=iso_number)
milestone=milestone['name'],
case_group=case_group,
iso_number=iso_number)
operation_systems = [{'name': config['name'], 'id': config['id']}
for config in project.get_config_by_name(
@ -215,7 +276,7 @@ def main():
plan_entries = [project.test_run_struct(
name='{case_group}'.format(case_group=case_group),
suite=case_group,
milestone=milestone,
milestone_id=milestone['id'],
description='Results of system tests ({case_group}) on iso # '
'"{iso_number}"'.format(case_group=case_group,
iso_number=iso_number),
@ -224,7 +285,7 @@ def main():
test_plan = project.add_plan(test_plan_name,
description='',
milestone=milestone,
milestone_id=milestone['id'],
entires=[
{
'suite_id': project.get_suite(
@ -240,19 +301,23 @@ def main():
logger.debug('Uploading tests results to TestRail...')
for os in operation_systems:
if 'centos' in os['name'].lower():
tests_results_centos = publish_results(project=project,
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_centos
)
tests_results_centos = publish_results(
project=project,
milestone_id=milestone['id'],
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_centos
)
if 'ubuntu' in os['name'].lower():
tests_results_ubuntu = publish_results(project=project,
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_ubuntu
)
tests_results_ubuntu = publish_results(
project=project,
milestone_id=milestone['id'],
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_ubuntu
)
logger.debug('Added new results for tests (CentOS): {tests}'.format(
tests=[r.group for r in tests_results_centos]

10
fuelweb_test/testrail/settings.py
View File

@ -27,7 +27,15 @@ JENKINS = {
}
class TestRailSettings():
class LaunchpadSettings(object):
project = os.environ.get('LAUNCHPAD_PROJECT', 'fuel')
milestone = os.environ.get('LAUNCHPAD_MILESTONE', '6.1')
closed_statuses = [
os.environ.get('LAUNCHPAD_RELEASED_STATUS', 'Fix Released')
]
class TestRailSettings(object):
url = os.environ.get('TESTRAIL_URL', 'https://mirantis.testrail.com')
user = os.environ.get('TESTRAIL_USER', 'user@example.com')
password = os.environ.get('TESTRAIL_PASSWORD', 'password')

47
fuelweb_test/testrail/testrail_client.py
View File

@ -30,12 +30,13 @@ class TestRailProject():
return project
return None
def test_run_struct(self, name, suite, milestone, description, config_ids,
include_all=True, assignedto=None, case_ids=None):
def test_run_struct(self, name, suite, milestone_id, description,
config_ids, include_all=True, assignedto=None,
case_ids=None):
struct = {
'name': name,
'suite_id': self.get_suite(suite)['id'],
'milestone_id': self.get_milestone(milestone)['id'],
'milestone_id': milestone_id,
'description': description,
'include_all': include_all,
'config_ids': config_ids
@ -130,6 +131,10 @@ class TestRailProject():
project_id=self.project['id'])
return self.client.send_get(plans_uri)
def get_plans_by_milestone(self, milestone_id):
plans = self.get_plans()
return [plan for plan in plans if plan['milestone_id'] == milestone_id]
def get_plan(self, plan_id):
plan_uri = 'get_plan/{plan_id}'.format(plan_id=plan_id)
return self.client.send_get(plan_uri)
@ -139,13 +144,13 @@ class TestRailProject():
if plan['name'] == name:
return self.get_plan(plan['id'])
def add_plan(self, name, description, milestone, entires):
def add_plan(self, name, description, milestone_id, entires):
add_plan_uri = 'add_plan/{project_id}'.format(
project_id=self.project['id'])
new_plan = {
'name': name,
'description': description,
'milestone_id': self.get_milestone(milestone)['id'],
'milestone_id': milestone_id,
'entries': entires
}
return self.client.send_post(add_plan_uri, new_plan)
@ -172,18 +177,27 @@ class TestRailProject():
if run['name'] == name:
return run
def get_previous_runs(self, milestone_id, config_id):
all_runs = []
for plan in self.get_plans_by_milestone(milestone_id=milestone_id):
run_ids = [run for run in
self.get_plan(plan_id=plan['id'])['entries'][0]['runs']
if config_id in run['config_ids']]
all_runs.extend(run_ids)
return all_runs
def add_run(self, new_run):
add_run_uri = 'add_run/{project_id}'.format(
project_id=self.project['id'])
return self.client.send_post(add_run_uri, new_run)
def update_run(self, name, milestone=None, description=None,
def update_run(self, name, milestone_id=None, description=None,
config_ids=None, include_all=None, case_ids=None):
tests_run = self.get_run(name)
update_run_uri = 'update_run/{run_id}'.format(run_id=tests_run['id'])
update_run = {}
if milestone:
update_run['milestone_id'] = self.get_milestone(milestone)['id']
if milestone_id:
update_run['milestone_id'] = milestone_id
if description:
update_run['description'] = description
if include_all is not None:
@ -194,18 +208,18 @@ class TestRailProject():
update_run['config_ids'] = config_ids
return self.client.send_post(update_run_uri, update_run)
def create_or_update_run(self, name, suite, milestone, description,
def create_or_update_run(self, name, suite, milestone_id, description,
config_ids, include_all=True, assignedto=None,
case_ids=None):
if self.get_run(name):
self.update_run(name=name,
milestone=milestone,
milestone_id=milestone_id,
description=description,
config_ids=config_ids,
include_all=include_all,
case_ids=case_ids)
else:
self.add_run(self.test_run_struct(name, suite, milestone,
self.add_run(self.test_run_struct(name, suite, milestone_id,
description, config_ids,
include_all=include_all,
assignedto=assignedto,
@ -258,6 +272,14 @@ class TestRailProject():
run_id=run_id, case_id=case_id)
return self.client.send_get(results_case_uri)
def get_all_results_for_case(self, run_ids, case_id):
all_results = []
for run_id in run_ids:
results = self.get_results_for_case(run_id=run_id,
case_id=case_id)
all_results.extend(results)
return all_results
def add_results_for_test(self, test_id, test_results):
add_results_test_uri = 'add_result/{test_id}'.format(test_id=test_id)
new_results = {
@ -281,7 +303,8 @@ class TestRailProject():
'status_id': self.get_status(results.status)['id'],
'comment': results.url,
'elapsed': results.duration,
'version': results.version
'version': results.version,
'custom_launchpad_bug': results.launchpad_bug
}
new_results['results'].append(new_result)
return self.client.send_post(add_results_test_uri, new_results)