openstack
/
fuel-qa
Code Issues Proposed changes

Merge "Create test plan when new swarm runner is started"

This commit is contained in:
Jenkins 2015-02-14 07:00:16 +00:00 committed by Gerrit Code Review
parent d5f9c00257 95891d69e0
commit c2c2580d00
6 changed files with 175 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fuelweb_test/requirements.txt
View File

@ -12,3 +12,4 @@ python-cinderclient>=1.0.5
python-neutronclient>=2.0
Jinja2
AllPairs==2.0.1
launchpadlib

3
fuelweb_test/testrail/builds.py
View File

@ -46,6 +46,9 @@ class Build():
if number == 'latest':
job_info = self.get_job_info(depth=0)
self.number = job_info["lastCompletedBuild"]["number"]
elif number == 'latest_started':
job_info = self.get_job_info(depth=0)
self.number = job_info["lastBuild"]["number"]
else:
self.number = int(number)

40
fuelweb_test/testrail/launchpad_client.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from launchpadlib.launchpad import Launchpad
class LaunchpadBug(object):
def __init__(self, bug_id):
self.launchpad = Launchpad.login_anonymously('just testing',
'production',
'.cache')
self.bug = self.launchpad.bugs[int(bug_id)]
@property
def targets(self):
return [{'project': str(task.bug_target_name).split('/')[0],
'milestone': str(task.milestone).split('/')[-1],
'status': str(task.status)} for task in self.bug_tasks]
def get_duplicate_of(self):
bug = self.bug
duplicates = []
while bug.duplicate_of and bug.id not in duplicates:
duplicates.append(bug.id)
bug = self.launchpad.load(str(bug.duplicate_of))
return LaunchpadBug(bug.id)
def __getattr__(self, item):
return self.bug.__getattr__(item)

109
fuelweb_test/testrail/report.py
View File

@ -22,6 +22,8 @@ from optparse import OptionParser
from builds import Build
from builds import get_jobs_for_view
from launchpad_client import LaunchpadBug
from settings import LaunchpadSettings
from settings import logger
from settings import TestRailSettings
from testrail_client import TestRailProject
@ -29,7 +31,7 @@ from testrail_client import TestRailProject
class TestResult():
def __init__(self, name, group, status, duration, url=None,
version=None, description=None):
version=None, description=None, launchpad_bug=None):
self.name = name
self.group = group
self._status = status
@ -37,6 +39,7 @@ class TestResult():
self.url = url
self.version = version
self.description = description
self.launchpad_bug = launchpad_bug
self.available_statuses = {
'passed': ['passed', 'fixed'],
'failed': ['failed', 'regression'],
@ -84,9 +87,9 @@ def retry(count=3):
return wrapped
def get_downstream_builds(jenkins_build_data):
return [{'name': b['jobName'], 'number': b['buildNumber']}
for b in jenkins_build_data['subBuilds']]
def get_downstream_builds(jenkins_build_data, status=None):
return [{'name': b['jobName'], 'number': b['buildNumber'],
'result': b['result']} for b in jenkins_build_data['subBuilds']]
def get_version(jenkins_build_data):
@ -120,9 +123,13 @@ def get_tests_results(systest_build):
return tests_results
def publish_results(project, test_plan, tests_suite, config_id, results):
def publish_results(project, milestone_id, test_plan,
tests_suite, config_id, results):
test_run_id = [run['id'] for run in test_plan['entries'][0]['runs'] if
config_id in run['config_ids']][0]
previous_tests_runs = project.get_previous_runs(milestone_id=milestone_id,
config_id=config_id)
cases = project.get_cases(suite=tests_suite)
tests = project.get_tests(run_id=test_run_id)
results_to_publish = []
@ -136,8 +143,18 @@ def publish_results(project, test_plan, tests_suite, config_id, results):
continue
existing_results_versions = [r['version'] for r in
project.get_results_for_test(test['id'])]
if result.version not in existing_results_versions:
results_to_publish.append(result)
if result.version in existing_results_versions:
continue
if result.status != 'passed':
run_ids = [run['id'] for run in previous_tests_runs]
case_id = project.get_case_by_group(suite=tests_suite,
group=result.group,
cases=cases)['id']
previous_results = project.get_all_results_for_case(
run_ids=run_ids,
case_id=case_id)
result.launchpad_bug = get_existing_bug_link(previous_results)
results_to_publish.append(result)
try:
if len(results_to_publish) > 0:
project.add_results_for_cases(run_id=test_run_id,
@ -151,6 +168,36 @@ def publish_results(project, test_plan, tests_suite, config_id, results):
return results_to_publish
@retry(count=3)
def get_existing_bug_link(previous_results):
results_with_bug = [result for result in previous_results if
result["custom_launchpad_bug"] is not None]
if not results_with_bug:
return
for result in sorted(results_with_bug,
key=lambda k: k['created_on'],
reverse=True):
try:
bug_id = int(result["custom_launchpad_bug"].strip('/').split(
'/')[-1])
except ValueError:
logger.warning('Link "{0}" doesn\'t contain bug id.'.format(
result["custom_launchpad_bug"]))
continue
try:
bug = LaunchpadBug(bug_id).get_duplicate_of()
except KeyError:
logger.error("Bug with id '{bug_id}' is private or "
"doesn't exist.".format(bug_id=bug_id))
return
for target in bug.targets:
if target['project'] == LaunchpadSettings.project and\
target['milestone'] == LaunchpadSettings.milestone and\
target['status'] not in LaunchpadSettings.closed_statuses:
return result["custom_launchpad_bug"]
def main():
parser = OptionParser(
@ -164,6 +211,8 @@ def main():
help='Jenkins swarm runner build number')
parser.add_option("-w", "--view", dest="jenkins_view", default=False,
help="Get system tests jobs from Jenkins view")
parser.add_option("-l", "--live", dest="live_report", action="store_true",
help="Get tests results from running swarm")
parser.add_option("-v", "--verbose",
action="store_true", dest="verbose", default=False,
help="Enable debug output")
@ -173,6 +222,9 @@ def main():
if options.verbose:
logger.setLevel(DEBUG)
if options.live_report:
options.build_number = 'latest_started'
tests_results_centos = []
tests_results_ubuntu = []
case_group = TestRailSettings.test_suite
@ -194,6 +246,11 @@ def main():
milestone, iso_number = get_version(runner_build.build_data)
for systest_build in tests_jobs:
if systest_build['result'] is None:
logger.debug("Skipping '{0}' job (build #{1}) because it's still "
"running...".format(systest_build['name'],
systest_build['number'],))
continue
if 'centos' in systest_build['name'].lower():
tests_results_centos.extend(get_tests_results(systest_build))
elif 'ubuntu' in systest_build['name'].lower():
@ -204,8 +261,12 @@ def main():
password=TestRailSettings.password,
project=TestRailSettings.project)
milestone = project.get_milestone(name=milestone)
test_plan_name = '{milestone} {case_group} iso #{iso_number}'.format(
milestone=milestone, case_group=case_group, iso_number=iso_number)
milestone=milestone['name'],
case_group=case_group,
iso_number=iso_number)
operation_systems = [{'name': config['name'], 'id': config['id']}
for config in project.get_config_by_name(
@ -215,7 +276,7 @@ def main():
plan_entries = [project.test_run_struct(
name='{case_group}'.format(case_group=case_group),
suite=case_group,
milestone=milestone,
milestone_id=milestone['id'],
description='Results of system tests ({case_group}) on iso # '
'"{iso_number}"'.format(case_group=case_group,
iso_number=iso_number),
@ -224,7 +285,7 @@ def main():
test_plan = project.add_plan(test_plan_name,
description='',
milestone=milestone,
milestone_id=milestone['id'],
entires=[
{
'suite_id': project.get_suite(
@ -240,19 +301,23 @@ def main():
logger.debug('Uploading tests results to TestRail...')
for os in operation_systems:
if 'centos' in os['name'].lower():
tests_results_centos = publish_results(project=project,
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_centos
)
tests_results_centos = publish_results(
project=project,
milestone_id=milestone['id'],
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_centos
)
if 'ubuntu' in os['name'].lower():
tests_results_ubuntu = publish_results(project=project,
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_ubuntu
)
tests_results_ubuntu = publish_results(
project=project,
milestone_id=milestone['id'],
test_plan=test_plan,
tests_suite=case_group,
config_id=os['id'],
results=tests_results_ubuntu
)
logger.debug('Added new results for tests (CentOS): {tests}'.format(
tests=[r.group for r in tests_results_centos]

10
fuelweb_test/testrail/settings.py
View File

@ -27,7 +27,15 @@ JENKINS = {
}
class TestRailSettings():
class LaunchpadSettings(object):
project = os.environ.get('LAUNCHPAD_PROJECT', 'fuel')
milestone = os.environ.get('LAUNCHPAD_MILESTONE', '6.1')
closed_statuses = [
os.environ.get('LAUNCHPAD_RELEASED_STATUS', 'Fix Released')
]
class TestRailSettings(object):
url = os.environ.get('TESTRAIL_URL', 'https://mirantis.testrail.com')
user = os.environ.get('TESTRAIL_USER', 'user@example.com')
password = os.environ.get('TESTRAIL_PASSWORD', 'password')

47
fuelweb_test/testrail/testrail_client.py
View File

@ -30,12 +30,13 @@ class TestRailProject():
return project
return None
def test_run_struct(self, name, suite, milestone, description, config_ids,
include_all=True, assignedto=None, case_ids=None):
def test_run_struct(self, name, suite, milestone_id, description,
config_ids, include_all=True, assignedto=None,
case_ids=None):
struct = {
'name': name,
'suite_id': self.get_suite(suite)['id'],
'milestone_id': self.get_milestone(milestone)['id'],
'milestone_id': milestone_id,
'description': description,
'include_all': include_all,
'config_ids': config_ids
@ -130,6 +131,10 @@ class TestRailProject():
project_id=self.project['id'])
return self.client.send_get(plans_uri)
def get_plans_by_milestone(self, milestone_id):
plans = self.get_plans()
return [plan for plan in plans if plan['milestone_id'] == milestone_id]
def get_plan(self, plan_id):
plan_uri = 'get_plan/{plan_id}'.format(plan_id=plan_id)
return self.client.send_get(plan_uri)
@ -139,13 +144,13 @@ class TestRailProject():
if plan['name'] == name:
return self.get_plan(plan['id'])
def add_plan(self, name, description, milestone, entires):
def add_plan(self, name, description, milestone_id, entires):
add_plan_uri = 'add_plan/{project_id}'.format(
project_id=self.project['id'])
new_plan = {
'name': name,
'description': description,
'milestone_id': self.get_milestone(milestone)['id'],
'milestone_id': milestone_id,
'entries': entires
}
return self.client.send_post(add_plan_uri, new_plan)
@ -172,18 +177,27 @@ class TestRailProject():
if run['name'] == name:
return run
def get_previous_runs(self, milestone_id, config_id):
all_runs = []
for plan in self.get_plans_by_milestone(milestone_id=milestone_id):
run_ids = [run for run in
self.get_plan(plan_id=plan['id'])['entries'][0]['runs']
if config_id in run['config_ids']]
all_runs.extend(run_ids)
return all_runs
def add_run(self, new_run):
add_run_uri = 'add_run/{project_id}'.format(
project_id=self.project['id'])
return self.client.send_post(add_run_uri, new_run)
def update_run(self, name, milestone=None, description=None,
def update_run(self, name, milestone_id=None, description=None,
config_ids=None, include_all=None, case_ids=None):
tests_run = self.get_run(name)
update_run_uri = 'update_run/{run_id}'.format(run_id=tests_run['id'])
update_run = {}
if milestone:
update_run['milestone_id'] = self.get_milestone(milestone)['id']
if milestone_id:
update_run['milestone_id'] = milestone_id
if description:
update_run['description'] = description
if include_all is not None:
@ -194,18 +208,18 @@ class TestRailProject():
update_run['config_ids'] = config_ids
return self.client.send_post(update_run_uri, update_run)
def create_or_update_run(self, name, suite, milestone, description,
def create_or_update_run(self, name, suite, milestone_id, description,
config_ids, include_all=True, assignedto=None,
case_ids=None):
if self.get_run(name):
self.update_run(name=name,
milestone=milestone,
milestone_id=milestone_id,
description=description,
config_ids=config_ids,
include_all=include_all,
case_ids=case_ids)
else:
self.add_run(self.test_run_struct(name, suite, milestone,
self.add_run(self.test_run_struct(name, suite, milestone_id,
description, config_ids,
include_all=include_all,
assignedto=assignedto,
@ -258,6 +272,14 @@ class TestRailProject():
run_id=run_id, case_id=case_id)
return self.client.send_get(results_case_uri)
def get_all_results_for_case(self, run_ids, case_id):
all_results = []
for run_id in run_ids:
results = self.get_results_for_case(run_id=run_id,
case_id=case_id)
all_results.extend(results)
return all_results
def add_results_for_test(self, test_id, test_results):
add_results_test_uri = 'add_result/{test_id}'.format(test_id=test_id)
new_results = {
@ -281,7 +303,8 @@ class TestRailProject():
'status_id': self.get_status(results.status)['id'],
'comment': results.url,
'elapsed': results.duration,
'version': results.version
'version': results.version,
'custom_launchpad_bug': results.launchpad_bug
}
new_results['results'].append(new_result)
return self.client.send_post(add_results_test_uri, new_results)