Add crm114 logstash queries to uncategorized page

Adding code to add in links to crm114 logstash queries
for those entries on the uncategorized page.

We only add a link if results were found during page
production.  This is to not waste human time on empty
queries.  However, this is a nice-to-have and can
be removed should any objection exist.

We also add some verbiage at the top of the uncategorized
page that describes what the crm114 links are as well
as including links to more in-depth docs, which are added
with change I086b2dfa68ca2239d70cec6db0b22951e6786ed7.

Change-Id: Ie572ef90a3393b43d28608b95a8a9b24bd014962
This commit is contained in:
pcrews 2014-08-12 22:00:02 -07:00
parent f3ef417122
commit c7b42bb88d
2 changed files with 29 additions and 5 deletions

View File

@ -15,8 +15,10 @@
# under the License. # under the License.
import argparse import argparse
import base64
import collections import collections
import datetime import datetime
import json
import operator import operator
import re import re
@ -72,7 +74,8 @@ def all_fails(classifier):
log = result.log_url.split("console.html")[0] log = result.log_url.split("console.html")[0]
all_fails["%s.%s" % (build, name)] = { all_fails["%s.%s" % (build, name)] = {
'log': log, 'log': log,
'timestamp': timestamp 'timestamp': timestamp,
'build_uuid': result.build_uuid
} }
return all_fails return all_fails
@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs):
return counts return counts
def classifying_rate(fails, data, engine): def classifying_rate(fails, data, engine, classifier):
"""Builds and prints the classification rate. """Builds and prints the classification rate.
It's important to know how good a job we are doing, so this It's important to know how good a job we are doing, so this
@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine):
key=lambda v: v['timestamp'], reverse=True) key=lambda v: v['timestamp'], reverse=True)
# Convert timestamp into string # Convert timestamp into string
for url in bad_job_urls[job]: for url in bad_job_urls[job]:
urlq = {}
url['timestamp'] = url['timestamp'].strftime( url['timestamp'] = url['timestamp'].strftime(
"%Y-%m-%dT%H:%M") "%Y-%m-%dT%H:%M")
# setup crm114 query for build_uuid
query = ('build_uuid: "%s" '
'AND error_pr:["-1000.0" TO "-10.0"] '
% url['build_uuid'])
urlq = dict(search=query,
fields=[],
offset=0,
timeframe=str(864000))
logstash_query = base64.urlsafe_b64encode(json.dumps(urlq))
logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query
results = classifier.hits_by_query(query, size=1)
if results:
url['crm114'] = logstash_url
classifying_rate = collections.defaultdict(int) classifying_rate = collections.defaultdict(int)
classifying_rate['overall'] = "%.1f" % ( classifying_rate['overall'] = "%.1f" % (
@ -231,7 +248,7 @@ def main():
fails = all_fails(classifier) fails = all_fails(classifier)
data = collect_metrics(classifier, fails) data = collect_metrics(classifier, fails)
engine = setup_template_engine(opts.templatedir) engine = setup_template_engine(opts.templatedir)
html = classifying_rate(fails, data, engine) html = classifying_rate(fails, data, engine, classifier)
if opts.output: if opts.output:
with open(opts.output, "w") as f: with open(opts.output, "w") as f:
f.write(html) f.write(html)

View File

@ -62,6 +62,10 @@
<li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li> <li><a href="#{{job[0]}}">{{job[0]}} ({{job[1]}})</a></li>
{% endfor %} {% endfor %}
</div> </div>
<div class='crm114-verbiage'>
The crm114 links are logstash queries showing log messages that have been flagged as potential errors.<br>
More information on the system can be found <a href="http://ci.openstack.org/logstash.html#crm114">here</a>
</div>
<div class="jobs"> <div class="jobs">
<h1>Unclassified failed jobs</h1> <h1>Unclassified failed jobs</h1>
Overall Categorization Rate: {{ rate['overall'] }}% Overall Categorization Rate: {{ rate['overall'] }}%
@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%
<h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2> <h2>{{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)</h2>
<ul> <ul>
{% for url in urls[job[0]] %} {% for url in urls[job[0]] %}
{% if url['crm114'] %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li> <li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a> : <a href="{{ url['crm114'] }}">crm114</a></li>
{% else %}
<li class="log-link">{{url['timestamp']}}: <a href="{{ url['log'] }}">{{ url['log'] }}</a></li>
{% endif %}
{% endfor %} {% endfor %}
</ul> </ul>
{% endfor %} {% endfor %}