[task] Rework junit-xml exporter
* show different tasks results separately * fix calculations PS: module rally.common.io.junit will be removed as soon as we clean CLI layer from old formats. Change-Id: Id950615e4e3f873d7928678278c030211e7cc998 Closes-Bug: #1711082
This commit is contained in:
Andrey Kurilin
parent
0d06ecd4d4
commit
e8ed18e11b
|
|
@ -827,3 +827,25 @@ class BackupHelper(object):
|
|||
if os.path.exists(path):
|
||||
LOG.debug("Deleting %s" % path)
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def prettify_xml(elem, level=0):
|
||||
"""Adds indents.
|
||||
|
||||
Code of this method was copied from
|
||||
http://effbot.org/zone/element-lib.htm#prettyprint
|
||||
|
||||
"""
|
||||
i = "\n" + level * " "
|
||||
if len(elem):
|
||||
if not elem.text or not elem.text.strip():
|
||||
elem.text = i + " "
|
||||
if not elem.tail or not elem.tail.strip():
|
||||
elem.tail = i
|
||||
for elem in elem:
|
||||
prettify_xml(elem, level + 1)
|
||||
if not elem.tail or not elem.tail.strip():
|
||||
elem.tail = i
|
||||
else:
|
||||
if level and (not elem.tail or not elem.tail.strip()):
|
||||
elem.tail = i
|
||||
|
|
|
|||
|
|
@ -12,10 +12,14 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime as dt
|
||||
import itertools
|
||||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from rally.common.io import junit
|
||||
from rally.common import utils
|
||||
from rally.common import version
|
||||
from rally import consts
|
||||
from rally.task import exporter
|
||||
|
||||
|
||||
|
|
@ -29,39 +33,87 @@ class JUnitXMLExporter(exporter.TaskExporter):
|
|||
|
||||
.. code-block:: xml
|
||||
|
||||
<testsuite errors="0"
|
||||
failures="0"
|
||||
name="Rally test suite"
|
||||
tests="1"
|
||||
time="29.97">
|
||||
<testcase classname="CinderVolumes"
|
||||
name="list_volumes"
|
||||
time="29.97" />
|
||||
</testsuite>
|
||||
<testsuites>
|
||||
<!--Report is generated by Rally 0.10.0 at 2017-06-04T05:14:00-->
|
||||
<testsuite id="task-uu-ii-dd"
|
||||
errors="0"
|
||||
failures="1"
|
||||
skipped="0"
|
||||
tests="2"
|
||||
time="75.0"
|
||||
timestamp="2017-06-04T05:14:00">
|
||||
<testcase classname="CinderVolumes"
|
||||
name="list_volumes"
|
||||
id="workload-1-uuid"
|
||||
time="29.9695231915"
|
||||
timestamp="2017-06-04T05:14:44" />
|
||||
<testcase classname="NovaServers"
|
||||
name="list_keypairs"
|
||||
id="workload-2-uuid"
|
||||
time="5"
|
||||
timestamp="2017-06-04T05:15:15">
|
||||
<failure>ooops</failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
"""
|
||||
|
||||
def generate(self):
|
||||
test_suite = junit.JUnit("Rally test suite")
|
||||
for task in self.tasks_results:
|
||||
root = ET.Element("testsuites")
|
||||
root.append(ET.Comment("Report is generated by Rally %s at %s" % (
|
||||
version.version_string(),
|
||||
dt.datetime.utcnow().strftime(consts.TimeFormat.ISO8601))))
|
||||
|
||||
for t in self.tasks_results:
|
||||
created_at = dt.datetime.strptime(t["created_at"],
|
||||
"%Y-%m-%dT%H:%M:%S")
|
||||
updated_at = dt.datetime.strptime(t["updated_at"],
|
||||
"%Y-%m-%dT%H:%M:%S")
|
||||
task = {
|
||||
"id": t["uuid"],
|
||||
"tests": 0,
|
||||
"errors": "0",
|
||||
"skipped": "0",
|
||||
"failures": 0,
|
||||
"time": "%.2f" % (updated_at - created_at).total_seconds(),
|
||||
"timestamp": t["created_at"],
|
||||
}
|
||||
test_cases = []
|
||||
for workload in itertools.chain(
|
||||
*[s["workloads"] for s in task["subtasks"]]):
|
||||
w_sla = workload["sla_results"].get("sla", [])
|
||||
*[s["workloads"] for s in t["subtasks"]]):
|
||||
class_name, name = workload["name"].split(".", 1)
|
||||
test_case = {
|
||||
"id": workload["uuid"],
|
||||
"time": "%.2f" % workload["full_duration"],
|
||||
"name": name,
|
||||
"classname": class_name,
|
||||
"timestamp": workload["created_at"]
|
||||
}
|
||||
if not workload["pass_sla"]:
|
||||
task["failures"] += 1
|
||||
test_case["failure"] = "\n".join(
|
||||
[s["detail"]
|
||||
for s in workload["sla_results"]["sla"]
|
||||
if not s["success"]])
|
||||
test_cases.append(test_case)
|
||||
|
||||
message = ",".join([sla["detail"] for sla in w_sla
|
||||
if not sla["success"]])
|
||||
if message:
|
||||
outcome = junit.JUnit.FAILURE
|
||||
else:
|
||||
outcome = junit.JUnit.SUCCESS
|
||||
test_suite.add_test(workload["name"],
|
||||
workload["full_duration"], outcome,
|
||||
message)
|
||||
task["tests"] = str(len(test_cases))
|
||||
task["failures"] = str(task["failures"])
|
||||
|
||||
result = test_suite.to_xml()
|
||||
testsuite = ET.SubElement(root, "testsuite", task)
|
||||
for test_case in test_cases:
|
||||
failure = test_case.pop("failure", None)
|
||||
test_case = ET.SubElement(testsuite, "testcase", test_case)
|
||||
if failure:
|
||||
ET.SubElement(test_case, "failure").text = failure
|
||||
|
||||
utils.prettify_xml(root)
|
||||
|
||||
raw_report = ET.tostring(root, encoding="utf-8").decode("utf-8")
|
||||
|
||||
if self.output_destination:
|
||||
return {"files": {self.output_destination: result},
|
||||
return {"files": {self.output_destination: raw_report},
|
||||
"open": "file://" + os.path.abspath(
|
||||
self.output_destination)}
|
||||
else:
|
||||
return {"print": result}
|
||||
return {"print": raw_report}
|
||||
|
|
|
|||
|
|
@ -18,9 +18,10 @@ import json
|
|||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from rally.common import utils
|
||||
from rally.common import version
|
||||
from rally import consts
|
||||
from rally.ui import utils
|
||||
from rally.ui import utils as ui_utils
|
||||
from rally.verification import reporter
|
||||
|
||||
|
||||
|
|
@ -297,7 +298,7 @@ class HTMLReporter(JSONReporter):
|
|||
# about the comparison strategy
|
||||
show_comparison_note = True
|
||||
|
||||
template = utils.get_template("verification/report.html")
|
||||
template = ui_utils.get_template("verification/report.html")
|
||||
context = {"uuids": uuids,
|
||||
"verifications": report["verifications"],
|
||||
"tests": report["tests"],
|
||||
|
|
@ -408,27 +409,6 @@ class JUnitXMLReporter(reporter.VerificationReporter):
|
|||
def validate(cls, output_destination):
|
||||
pass
|
||||
|
||||
def _prettify_xml(self, elem, level=0):
|
||||
"""Adds indents.
|
||||
|
||||
Code of this method was copied from
|
||||
http://effbot.org/zone/element-lib.htm#prettyprint
|
||||
|
||||
"""
|
||||
i = "\n" + level * " "
|
||||
if len(elem):
|
||||
if not elem.text or not elem.text.strip():
|
||||
elem.text = i + " "
|
||||
if not elem.tail or not elem.tail.strip():
|
||||
elem.tail = i
|
||||
for elem in elem:
|
||||
self._prettify_xml(elem, level + 1)
|
||||
if not elem.tail or not elem.tail.strip():
|
||||
elem.tail = i
|
||||
else:
|
||||
if level and (not elem.tail or not elem.tail.strip()):
|
||||
elem.tail = i
|
||||
|
||||
def generate(self):
|
||||
root = ET.Element("testsuites")
|
||||
|
||||
|
|
@ -495,7 +475,7 @@ class JUnitXMLReporter(reporter.VerificationReporter):
|
|||
# wtf is it?! we should add validation of results...
|
||||
pass
|
||||
|
||||
self._prettify_xml(root)
|
||||
utils.prettify_xml(root)
|
||||
|
||||
raw_report = ET.tostring(root, encoding="utf-8").decode("utf-8")
|
||||
if self.output_destination:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
<testsuites>
|
||||
<!--Report is generated by Rally $VERSION at $TIME-->
|
||||
<testsuite errors="0" failures="1" id="task-uu-ii-dd" skipped="0" tests="2" time="75.00" timestamp="2017-06-04T05:14:00">
|
||||
<testcase classname="CinderVolumes" id="workload-1-uuid" name="list_volumes" time="29.97" timestamp="2017-06-04T05:14:44" />
|
||||
<testcase classname="NovaServers" id="workload-2-uuid" name="list_keypairs" time="5.00" timestamp="2017-06-04T05:15:15">
|
||||
<failure>ooops</failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
|
@ -12,70 +12,82 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime as dt
|
||||
import os
|
||||
|
||||
import mock
|
||||
|
||||
from rally.plugins.common.exporters import junit
|
||||
from tests.unit import test
|
||||
|
||||
|
||||
def get_tasks_results():
|
||||
task_id = "2fa4f5ff-7d23-4bb0-9b1f-8ee235f7f1c8"
|
||||
workload = {"created_at": "2017-06-04T05:14:44",
|
||||
"updated_at": "2017-06-04T05:15:14",
|
||||
"task_uuid": task_id,
|
||||
"position": 0,
|
||||
"name": "CinderVolumes.list_volumes",
|
||||
"description": "List all volumes.",
|
||||
"data": {"raw": []},
|
||||
"full_duration": 29.969523191452026,
|
||||
"sla": {},
|
||||
"sla_results": {"sla": []},
|
||||
"load_duration": 2.03029203414917,
|
||||
"hooks": [],
|
||||
"id": 3}
|
||||
task = {"subtasks": [
|
||||
{"task_uuid": task_id,
|
||||
"workloads": [workload]}]}
|
||||
return [task]
|
||||
return [{
|
||||
"uuid": "task-uu-ii-dd",
|
||||
"created_at": "2017-06-04T05:14:00",
|
||||
"updated_at": "2017-06-04T05:15:15",
|
||||
"subtasks": [
|
||||
{"task_uuid": task_id,
|
||||
"workloads": [
|
||||
{
|
||||
"uuid": "workload-1-uuid",
|
||||
"created_at": "2017-06-04T05:14:44",
|
||||
"updated_at": "2017-06-04T05:15:14",
|
||||
"task_uuid": task_id,
|
||||
"position": 0,
|
||||
"name": "CinderVolumes.list_volumes",
|
||||
"full_duration": 29.969523191452026,
|
||||
"sla_results": {"sla": []},
|
||||
"pass_sla": True
|
||||
},
|
||||
{
|
||||
"uuid": "workload-2-uuid",
|
||||
"created_at": "2017-06-04T05:15:15",
|
||||
"updated_at": "2017-06-04T05:16:14",
|
||||
"task_uuid": task_id,
|
||||
"position": 1,
|
||||
"name": "NovaServers.list_keypairs",
|
||||
"full_duration": 5,
|
||||
"sla_results": {"sla": [
|
||||
{"criterion": "Failing",
|
||||
"success": False,
|
||||
"detail": "ooops"},
|
||||
{"criterion": "Ok",
|
||||
"success": True,
|
||||
"detail": None},
|
||||
]},
|
||||
"pass_sla": False
|
||||
},
|
||||
]}]}]
|
||||
|
||||
|
||||
class JUnitXMLExporterTestCase(test.TestCase):
|
||||
def setUp(self):
|
||||
super(JUnitXMLExporterTestCase, self).setUp()
|
||||
self.datetime = dt.datetime
|
||||
|
||||
def test_generate(self):
|
||||
content = ("<testsuite errors=\"0\""
|
||||
" failures=\"0\""
|
||||
" name=\"Rally test suite\""
|
||||
" tests=\"1\""
|
||||
" time=\"29.97\">"
|
||||
"<testcase classname=\"CinderVolumes\""
|
||||
" name=\"list_volumes\""
|
||||
" time=\"29.97\" />"
|
||||
"</testsuite>")
|
||||
patcher = mock.patch("rally.plugins.common.exporters.junit.dt")
|
||||
self.dt = patcher.start()
|
||||
self.dt.datetime.strptime.side_effect = self.datetime.strptime
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
@mock.patch("rally.plugins.common.exporters.junit.version.version_string")
|
||||
def test_generate(self, mock_version_string):
|
||||
now = self.dt.datetime.utcnow.return_value
|
||||
now.strftime.return_value = "$TIME"
|
||||
mock_version_string.return_value = "$VERSION"
|
||||
|
||||
with open(os.path.join(os.path.dirname(__file__),
|
||||
"junit_report.xml")) as f:
|
||||
expected_report = f.read()
|
||||
|
||||
reporter = junit.JUnitXMLExporter(get_tasks_results(),
|
||||
output_destination=None)
|
||||
self.assertEqual({"print": content}, reporter.generate())
|
||||
self.assertEqual({"print": expected_report}, reporter.generate())
|
||||
|
||||
reporter = junit.JUnitXMLExporter(get_tasks_results(),
|
||||
output_destination="path")
|
||||
self.assertEqual({"files": {"path": content},
|
||||
self.assertEqual({"files": {"path": expected_report},
|
||||
"open": "file://" + os.path.abspath("path")},
|
||||
reporter.generate())
|
||||
|
||||
def test_generate_fail(self):
|
||||
tasks_results = get_tasks_results()
|
||||
tasks_results[0]["subtasks"][0]["workloads"][0]["sla_results"] = {
|
||||
"sla": [{"success": False, "detail": "error"}]}
|
||||
content = ("<testsuite errors=\"0\""
|
||||
" failures=\"1\""
|
||||
" name=\"Rally test suite\""
|
||||
" tests=\"1\""
|
||||
" time=\"29.97\">"
|
||||
"<testcase classname=\"CinderVolumes\""
|
||||
" name=\"list_volumes\""
|
||||
" time=\"29.97\">"
|
||||
"<failure message=\"error\" /></testcase>"
|
||||
"</testsuite>")
|
||||
reporter = junit.JUnitXMLExporter(tasks_results,
|
||||
output_destination=None)
|
||||
self.assertEqual({"print": content}, reporter.generate())
|
||||
|
|
|
|||
|
|
@ -287,13 +287,13 @@ class JSONReporterTestCase(test.TestCase):
|
|||
|
||||
@ddt.ddt
|
||||
class HTMLReporterTestCase(test.TestCase):
|
||||
@mock.patch("%s.utils" % PATH)
|
||||
@mock.patch("%s.ui_utils" % PATH)
|
||||
@mock.patch("%s.json.dumps" % PATH)
|
||||
@ddt.data((reporters.HTMLReporter, False),
|
||||
(reporters.HTMLStaticReporter, True))
|
||||
@ddt.unpack
|
||||
def test_generate(self, cls, include_libs, mock_dumps, mock_utils):
|
||||
mock_render = mock_utils.get_template.return_value.render
|
||||
def test_generate(self, cls, include_libs, mock_dumps, mock_ui_utils):
|
||||
mock_render = mock_ui_utils.get_template.return_value.render
|
||||
|
||||
reporter = cls(get_verifications(), None)
|
||||
|
||||
|
|
@ -301,7 +301,7 @@ class HTMLReporterTestCase(test.TestCase):
|
|||
reporter.generate())
|
||||
mock_render.assert_called_once_with(data=mock_dumps.return_value,
|
||||
include_libs=include_libs)
|
||||
mock_utils.get_template.assert_called_once_with(
|
||||
mock_ui_utils.get_template.assert_called_once_with(
|
||||
"verification/report.html")
|
||||
|
||||
self.assertEqual(1, mock_dumps.call_count)
|
||||
|
|
|
|||
Loading…
Reference in New Issue