diff --git a/elastic_recheck/cmd/uncategorized_fails.py b/elastic_recheck/cmd/uncategorized_fails.py index 64afde1e..3ef30d6b 100755 --- a/elastic_recheck/cmd/uncategorized_fails.py +++ b/elastic_recheck/cmd/uncategorized_fails.py @@ -15,8 +15,10 @@ # under the License. import argparse +import base64 import collections import datetime +import json import operator import re @@ -72,7 +74,8 @@ def all_fails(classifier): log = result.log_url.split("console.html")[0] all_fails["%s.%s" % (build, name)] = { 'log': log, - 'timestamp': timestamp + 'timestamp': timestamp, + 'build_uuid': result.build_uuid } return all_fails @@ -85,7 +88,7 @@ def num_fails_per_build_name(all_jobs): return counts -def classifying_rate(fails, data, engine): +def classifying_rate(fails, data, engine, classifier): """Builds and prints the classification rate. It's important to know how good a job we are doing, so this @@ -121,8 +124,22 @@ def classifying_rate(fails, data, engine): key=lambda v: v['timestamp'], reverse=True) # Convert timestamp into string for url in bad_job_urls[job]: + urlq = {} url['timestamp'] = url['timestamp'].strftime( "%Y-%m-%dT%H:%M") + # setup crm114 query for build_uuid + query = ('build_uuid: "%s" ' + 'AND error_pr:["-1000.0" TO "-10.0"] ' + % url['build_uuid']) + urlq = dict(search=query, + fields=[], + offset=0, + timeframe=str(864000)) + logstash_query = base64.urlsafe_b64encode(json.dumps(urlq)) + logstash_url = 'http://logstash.openstack.org/#%s' % logstash_query + results = classifier.hits_by_query(query, size=1) + if results: + url['crm114'] = logstash_url classifying_rate = collections.defaultdict(int) classifying_rate['overall'] = "%.1f" % ( @@ -231,7 +248,7 @@ def main(): fails = all_fails(classifier) data = collect_metrics(classifier, fails) engine = setup_template_engine(opts.templatedir) - html = classifying_rate(fails, data, engine) + html = classifying_rate(fails, data, engine, classifier) if opts.output: with open(opts.output, "w") as f: f.write(html) diff --git a/web/share/templates/uncategorized.html b/web/share/templates/uncategorized.html index 82f1478a..04060000 100644 --- a/web/share/templates/uncategorized.html +++ b/web/share/templates/uncategorized.html @@ -62,6 +62,10 @@
  • {{job[0]}} ({{job[1]}})
  • {% endfor %} +
    +The crm114 links are logstash queries showing log messages that have been flagged as potential errors.
    +More information on the system can be found here +

    Unclassified failed jobs

    Overall Categorization Rate: {{ rate['overall'] }}% @@ -81,8 +85,11 @@ Overall Categorization Rate: {{ rate['overall'] }}%

    {{ job[0] }} : {{ job[1] }} Uncategorized Fails. {{rate[job[0]]}}% Classification Rate ({{total_job_failures[job[0]]}} Total Fails)

    {% endfor %}