add uncategorized failure generation code
the following adds a new tool to build a web page that will give us a full list of all the uncategorized failures and their logs. This gives us a todo list to start building ER queries from. this will require an infrastructure change to run this on cron to generate a public page. it also brings in jinja2 for html templating, which will be useful for future html generation jobs. Change-Id: I5114296fbac2cde9c6b0133e2717e1cc28fb631d
This commit is contained in:
Sean Dague
parent
8f8b23ab88
commit
c96a8e8e6d
|
|
@ -0,0 +1,210 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2014 Samsung Electronics. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import operator
|
||||
import re
|
||||
|
||||
import jinja2
|
||||
|
||||
import elastic_recheck.elasticRecheck as er
|
||||
import elastic_recheck.results as er_results
|
||||
|
||||
|
||||
def get_options():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='''Build the list of all uncategorized test runs.
|
||||
|
||||
Note: This will take a few minutes to run.''')
|
||||
parser.add_argument('--dir', '-d', help="Queries Directory",
|
||||
default="queries")
|
||||
parser.add_argument('-t', '--templatedir', help="Template Directory")
|
||||
parser.add_argument('-o', '--output', help="Output File")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def setup_template_engine(directory):
|
||||
path = ["web/share/templates"]
|
||||
if directory:
|
||||
path.append(directory)
|
||||
|
||||
loader = jinja2.FileSystemLoader(path)
|
||||
env = jinja2.Environment(loader=loader)
|
||||
return env.get_template("uncategorized.html")
|
||||
|
||||
|
||||
def all_fails(classifier):
|
||||
"""Find all the the fails in the integrated gate.
|
||||
|
||||
This attempts to find all the build jobs in the integrated gate
|
||||
so we can figure out how good we are doing on total classification.
|
||||
"""
|
||||
all_fails = {}
|
||||
query = ('filename:"console.html" '
|
||||
'AND message:"Finished: FAILURE" '
|
||||
'AND build_queue:"gate"')
|
||||
results = classifier.hits_by_query(query, size=30000)
|
||||
facets = er_results.FacetSet()
|
||||
facets.detect_facets(results, ["build_uuid"])
|
||||
for build in facets:
|
||||
for result in facets[build]:
|
||||
# not perfect, but basically an attempt to show the integrated
|
||||
# gate. Would be nice if there was a zuul attr for this in es.
|
||||
if re.search("(^openstack/|devstack|grenade)", result.project):
|
||||
name = result.build_name
|
||||
log = result.log_url.split("console.html")[0]
|
||||
all_fails["%s.%s" % (build, name)] = log
|
||||
return all_fails
|
||||
|
||||
|
||||
def num_fails_per_build_name(all_jobs):
|
||||
counts = collections.defaultdict(int)
|
||||
for f in all_jobs:
|
||||
build, job = f.split('.', 1)
|
||||
counts[job] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def classifying_rate(fails, data, engine):
|
||||
"""Builds and prints the classification rate.
|
||||
|
||||
It's important to know how good a job we are doing, so this
|
||||
tool runs through all the failures we've got and builds the
|
||||
classification rate. For every failure in the gate queue did
|
||||
we find a match for it.
|
||||
"""
|
||||
found_fails = {k: False for (k, v) in fails.iteritems()}
|
||||
|
||||
for bugnum in data:
|
||||
bug = data[bugnum]
|
||||
for job in bug['failed_jobs']:
|
||||
found_fails[job] = True
|
||||
|
||||
total = len(found_fails.keys())
|
||||
bad_jobs = collections.defaultdict(int)
|
||||
bad_job_urls = collections.defaultdict(list)
|
||||
count = 0
|
||||
for f in fails:
|
||||
if found_fails[f] is True:
|
||||
count += 1
|
||||
else:
|
||||
build, job = f.split('.', 1)
|
||||
bad_jobs[job] += 1
|
||||
bad_job_urls[job].append(fails[f])
|
||||
|
||||
classifying_rate = ((float(count) / float(total)) * 100.0)
|
||||
sort = sorted(
|
||||
bad_jobs.iteritems(),
|
||||
key=operator.itemgetter(1),
|
||||
reverse=True)
|
||||
|
||||
tvars = {
|
||||
"rate": classifying_rate,
|
||||
"jobs": sort,
|
||||
"urls": bad_job_urls
|
||||
}
|
||||
return engine.render(tvars)
|
||||
|
||||
|
||||
def _status_count(results):
|
||||
counts = {}
|
||||
facets = er_results.FacetSet()
|
||||
facets.detect_facets(
|
||||
results,
|
||||
["build_status", "build_uuid"])
|
||||
|
||||
for key in facets:
|
||||
counts[key] = len(facets[key])
|
||||
return counts
|
||||
|
||||
|
||||
def _failure_count(hits):
|
||||
if "FAILURE" in hits:
|
||||
return hits["FAILURE"]
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def _failed_jobs(results):
|
||||
failed_jobs = []
|
||||
facets = er_results.FacetSet()
|
||||
facets.detect_facets(
|
||||
results,
|
||||
["build_status", "build_uuid"])
|
||||
if "FAILURE" in facets:
|
||||
for build in facets["FAILURE"]:
|
||||
for result in facets["FAILURE"][build]:
|
||||
failed_jobs.append("%s.%s" % (build, result.build_name))
|
||||
return failed_jobs
|
||||
|
||||
|
||||
def _count_fails_per_build_name(hits):
|
||||
facets = er_results.FacetSet()
|
||||
counts = collections.defaultdict(int)
|
||||
facets.detect_facets(
|
||||
hits,
|
||||
["build_status", "build_name", "build_uuid"])
|
||||
if "FAILURE" in facets:
|
||||
for build_name in facets["FAILURE"]:
|
||||
counts[build_name] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def _failure_percentage(hits, fails):
|
||||
total_fails_per_build_name = num_fails_per_build_name(fails)
|
||||
fails_per_build_name = _count_fails_per_build_name(hits)
|
||||
per = {}
|
||||
for build in fails_per_build_name:
|
||||
this_job = fails_per_build_name[build]
|
||||
if build in total_fails_per_build_name:
|
||||
total = total_fails_per_build_name[build]
|
||||
per[build] = (float(this_job) / float(total)) * 100.0
|
||||
return per
|
||||
|
||||
|
||||
def collect_metrics(classifier, fails):
|
||||
data = {}
|
||||
for q in classifier.queries:
|
||||
results = classifier.hits_by_query(q['query'], size=30000)
|
||||
hits = _status_count(results)
|
||||
data[q['bug']] = {
|
||||
'fails': _failure_count(hits),
|
||||
'hits': hits,
|
||||
'percentages': _failure_percentage(results, fails),
|
||||
'query': q['query'],
|
||||
'failed_jobs': _failed_jobs(results)
|
||||
}
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def main():
|
||||
opts = get_options()
|
||||
classifier = er.Classifier(opts.dir)
|
||||
fails = all_fails(classifier)
|
||||
data = collect_metrics(classifier, fails)
|
||||
engine = setup_template_engine(opts.templatedir)
|
||||
html = classifying_rate(fails, data, engine)
|
||||
if opts.output:
|
||||
with open(opts.output, "w") as f:
|
||||
f.write(html)
|
||||
else:
|
||||
print html
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -7,3 +7,4 @@ lockfile
|
|||
pbr>=0.5.21,<1.0
|
||||
Babel>=0.9.6
|
||||
launchpadlib
|
||||
Jinja2
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ console_scripts =
|
|||
elastic-recheck = elastic_recheck.bot:main
|
||||
elastic-recheck-graph = elastic_recheck.cmd.graph:main
|
||||
elastic-recheck-success = elastic_recheck.cmd.check_success:main
|
||||
elastic-recheck-uncategorized = elastic_recheck.cmd.uncategorized_fails:main
|
||||
|
||||
[build_sphinx]
|
||||
source-dir = doc/source
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
<html xmlns="http://www.w3.org/1999/xhtml"
|
||||
xmlns:py="http://genshi.edgewall.org/"
|
||||
lang="en">
|
||||
<HEAD>
|
||||
<TITLE>Elastic Recheck</TITLE>
|
||||
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/jquery.min.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/jquery-visibility.min.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/jquery-graphite.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/common.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/jquery.flot.min.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="http://status.openstack.org/jquery.flot.time.min.js"></script>
|
||||
<script type="text/javascript"
|
||||
src="elastic-recheck.js"></script>
|
||||
|
||||
<!-- Google Fonts -->
|
||||
<link href='http://fonts.googleapis.com/css?family=PT+Sans&subset=latin' rel='stylesheet' type='text/css'/>
|
||||
|
||||
<!-- Framework CSS -->
|
||||
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/screen.css" type="text/css" media="screen, projection"/>
|
||||
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/blueprint/print.css" type="text/css" media="print"/>
|
||||
|
||||
<!-- IE CSS -->
|
||||
<!--[if lt IE 8]><link rel="stylesheet" href="http://www.openstack.org/blueprint/ie.css" type="text/css" media="screen, projection"><![endif]-->
|
||||
|
||||
<!-- OpenStack Specific CSS -->
|
||||
|
||||
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/dropdown.css" type="text/css" media="screen, projection, print"/>
|
||||
|
||||
<!-- Page Specific CSS -->
|
||||
<link rel="stylesheet" href="http://www.openstack.org/themes/openstack/css/home.css" type="text/css" media="screen, projection, print"/>
|
||||
|
||||
<link rel="stylesheet" type="text/css" href="http://www.openstack.org/themes/openstack/css/main.css" />
|
||||
|
||||
|
||||
<style type="text/css">
|
||||
.graph {
|
||||
width: 600px;
|
||||
height: 200px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.extlink {
|
||||
margin-left: 2em;
|
||||
margin-right: 2em;
|
||||
}
|
||||
.bug-container {
|
||||
margin-bottom: 2em;
|
||||
}
|
||||
table {
|
||||
width: auto;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<script type="text/javascript">
|
||||
</script>
|
||||
|
||||
</HEAD>
|
||||
|
||||
<BODY>
|
||||
<script type="text/javascript">header('Rechecks');</script>
|
||||
<div id="main-container" class="container">
|
||||
{% block body %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
|
||||
<script type="text/javascript">footer();</script>
|
||||
</BODY>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
{% extends "base.html" %}
|
||||
{% block body %}
|
||||
{{ super() }}
|
||||
|
||||
<style>
|
||||
.menu {
|
||||
float: right;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="menu">
|
||||
<a name="top"></a>
|
||||
{% for job in jobs %}
|
||||
<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="jobs">
|
||||
<h1>Unclassified failed jobs</h1>
|
||||
Overall Categorization Rate: {{ rate }}
|
||||
|
||||
|
||||
{% for job in jobs %}
|
||||
<a name="{{job[0]}}"></a>
|
||||
<a href="#top"><i>back to top</i></a>
|
||||
<h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails</h2>
|
||||
<ul>
|
||||
{% for url in urls[job[0]] %}
|
||||
<li><a href="{{ url }}">{{ url }}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endfor %}
|
||||
{% endblock %}
|
||||
</div>
|
||||
Loading…
Reference in New Issue