From 97f6408b540c1799cecfad84caf831d77e21f602 Mon Sep 17 00:00:00 2001 From: David Moreau-Simard Date: Tue, 3 Oct 2017 20:08:31 -0400 Subject: [PATCH] Add support for Zuulv3-specific parameters in elastic-recheck This commit ensures elastic-recheck is able to support zuul v2 and v3 simultaneously: - Add message queries based on v3 completion messages - Include job-output.txt where console.html was expected Change-Id: If3d7990d892a9698a112d9ff1dd5160998a4efe6 Depends-On: I7e34206d7968bf128e140468b9a222ecbce3a8f1 --- elastic_recheck/cmd/check_success.py | 8 ++---- elastic_recheck/cmd/uncategorized_fails.py | 10 +++++-- elastic_recheck/config.py | 15 ++++++---- elastic_recheck/elasticRecheck.py | 6 ++-- elastic_recheck/query_builder.py | 32 ++++++++++++++-------- queries/1260311.yaml | 2 +- queries/1260654.yaml | 2 +- queries/1269940.yaml | 2 +- queries/1270309.yaml | 2 +- queries/1384373.yaml | 2 +- queries/1695970.yaml | 2 +- queries/1718990.yaml | 2 +- 12 files changed, 52 insertions(+), 33 deletions(-) diff --git a/elastic_recheck/cmd/check_success.py b/elastic_recheck/cmd/check_success.py index c27bbdc7..a47a4a1a 100755 --- a/elastic_recheck/cmd/check_success.py +++ b/elastic_recheck/cmd/check_success.py @@ -24,6 +24,7 @@ import time from launchpadlib import launchpad +import elastic_recheck.config as er_config import elastic_recheck.elasticRecheck as er import elastic_recheck.results as er_results @@ -51,17 +52,14 @@ def all_fails(classifier): so we can figure out how good we are doing on total classification. """ all_fails = {} - query = ('filename:"console.html" ' - 'AND message:"Finished: FAILURE" ' - 'AND build_queue:"gate"') - results = classifier.hits_by_query(query, size=30000) + results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000) facets = er_results.FacetSet() facets.detect_facets(results, ["build_uuid"]) for build in facets: for result in facets[build]: # not perfect, but basically an attempt to show the integrated # gate. Would be nice if there was a zuul attr for this in es. - if re.search("(^openstack/|devstack|grenade)", result.project): + if re.search(er_config.INCLUDED_PROJECTS_REGEX, result.project): all_fails["%s.%s" % (build, result.build_name)] = False return all_fails diff --git a/elastic_recheck/cmd/uncategorized_fails.py b/elastic_recheck/cmd/uncategorized_fails.py index 54a2eb55..40adbb82 100755 --- a/elastic_recheck/cmd/uncategorized_fails.py +++ b/elastic_recheck/cmd/uncategorized_fails.py @@ -125,7 +125,10 @@ def all_fails(classifier, config=None): if result.project in integrated_gate_projects: name = result.build_name timestamp = dp.parse(result.timestamp) - log = result.log_url.split("console.html")[0] + if 'console.html' in result.log_url: + log = result.log_url.split('console.html')[0] + elif 'job-output.txt' in result.log_url: + log = result.log_url.split('job-output.txt')[0] integrated_fails["%s.%s" % (build, name)] = { 'log': log, 'timestamp': timestamp, @@ -137,7 +140,10 @@ def all_fails(classifier, config=None): if re.search(config.included_projects_regex, result.project): name = result.build_name timestamp = dp.parse(result.timestamp) - log = result.log_url.split("console.html")[0] + if 'console.html' in result.log_url: + log = result.log_url.split('console.html')[0] + elif 'job-output.txt' in result.log_url: + log = result.log_url.split('job-output.txt')[0] other_fails["%s.%s" % (build, name)] = { 'log': log, 'timestamp': timestamp, diff --git a/elastic_recheck/config.py b/elastic_recheck/config.py index d4cef0ba..798e737a 100644 --- a/elastic_recheck/config.py +++ b/elastic_recheck/config.py @@ -47,11 +47,16 @@ EXCLUDED_JOBS_REGEX = re.compile('(' + '|'.join(EXCLUDED_JOBS) + ')') INCLUDED_PROJECTS_REGEX = "(^openstack/|devstack|grenade)" -ALL_FAILS_QUERY = ('filename:"console.html" ' - 'AND (message:"Finished: FAILURE" ' - 'OR message:"[Zuul] Job complete, result: FAILURE") ' - 'AND build_queue:"gate" ' - 'AND voting:"1"') +# TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported +# Let's value legibility over pep8 line width here... +ALL_FAILS_QUERY = ( + '((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")' # flake8: noqa + ' OR ' + '(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))' # flake8: noqa + ' AND build_status:"FAILURE"' + ' AND build_queue:"gate"' + ' AND voting:"1"' +) UNCAT_MAX_SEARCH_SIZE = 30000 diff --git a/elastic_recheck/elasticRecheck.py b/elastic_recheck/elasticRecheck.py index ce6730b7..d315cc51 100644 --- a/elastic_recheck/elasticRecheck.py +++ b/elastic_recheck/elasticRecheck.py @@ -31,7 +31,7 @@ from elastic_recheck import results def required_files(job): - files = ['console.html'] + files = [] if re.match("tempest-dsvm", job): files.extend([ 'logs/screen-n-api.txt', @@ -246,9 +246,11 @@ class Stream(object): query = qb.files_ready(change, patch, name, build_short_uuid) r = self.es.search(query, size='80', recent=True) files = [x['term'] for x in r.terms] + # TODO(dmsimard): Reliably differentiate zuul v2 and v3 jobs required = required_files(name) missing_files = [x for x in required if x not in files] - if len(missing_files) != 0: + if (len(missing_files) != 0 or + ('console.html' not in files and 'job-output.txt' not in files)): msg = ("%s missing for %s %s,%s,%s" % ( missing_files, name, change, patch, build_short_uuid)) raise FilesNotReady(msg) diff --git a/elastic_recheck/query_builder.py b/elastic_recheck/query_builder.py index 1e3e0996..a340e2b7 100644 --- a/elastic_recheck/query_builder.py +++ b/elastic_recheck/query_builder.py @@ -69,22 +69,30 @@ def single_queue(query, queue, facet=None): (query, queue), facet=facet) -def result_ready(review, patch, name, build_short_uuid): +def result_ready(change, patchset, name, short_uuid): """A query to determine if we have a failure for a particular patch. This is looking for a particular FAILURE line in the console log, which lets us know that we've got results waiting that we need to process. """ - return generic('filename:"console.html" AND ' - '(message:"[SCP] Copying console log" ' - 'OR message:"Grabbing consoleLog" ' - 'OR message:"[Zuul] Job complete") ' - 'AND build_status:"FAILURE" ' - 'AND build_change:"%s" ' - 'AND build_patchset:"%s" ' - 'AND build_name:"%s" ' - 'AND build_short_uuid:%s' % - (review, patch, name, build_short_uuid)) + # TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported + # Let's value legibility over pep8 line width here... + query = ( + '((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")' # flake8: noqa + ' OR ' + '(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))' # flake8: noqa + ' AND build_status:"FAILURE"' + ' AND build_change:"{change}"' + ' AND build_patchset:"{patchset}"' + ' AND build_name:"{name}"' + ' AND build_short_uuid:"{short_uuid}"' + ) + return generic(query.format( + change=change, + patchset=patchset, + name=name, + short_uuid=short_uuid + )) def files_ready(review, patch, name, build_short_uuid): @@ -119,7 +127,7 @@ def single_patch(query, review, patch, build_short_uuid): def most_recent_event(): return generic( - 'filename:console.html ' + '(filename:"console.html" OR filename:"job-output.txt") ' 'AND (build_queue:gate OR build_queue:check) ' 'AND NOT tags:_grokparsefailure ' 'AND NOT message:"%{logmessage}" ') diff --git a/queries/1260311.yaml b/queries/1260311.yaml index d184758e..ad0f048d 100644 --- a/queries/1260311.yaml +++ b/queries/1260311.yaml @@ -1,4 +1,4 @@ query: > message:"java.io.InterruptedIOException" - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") suppress-graph: true diff --git a/queries/1260654.yaml b/queries/1260654.yaml index b6244b69..05645a00 100644 --- a/queries/1260654.yaml +++ b/queries/1260654.yaml @@ -2,5 +2,5 @@ query: > message:"java.io.IOException" AND message:"Remote call on" AND message:"failed" - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") suppress-graph: true diff --git a/queries/1269940.yaml b/queries/1269940.yaml index 6948621a..26034faa 100644 --- a/queries/1269940.yaml +++ b/queries/1269940.yaml @@ -1,4 +1,4 @@ query: > message:"[EnvInject] - [ERROR] - SEVERE ERROR occurs:" - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") suppress-graph: true diff --git a/queries/1270309.yaml b/queries/1270309.yaml index 4188d48d..119a8713 100644 --- a/queries/1270309.yaml +++ b/queries/1270309.yaml @@ -1,4 +1,4 @@ query: > message:"java.lang.InterruptedException" - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") suppress-graph: true diff --git a/queries/1384373.yaml b/queries/1384373.yaml index cba221dd..fee22653 100644 --- a/queries/1384373.yaml +++ b/queries/1384373.yaml @@ -1,3 +1,3 @@ query: > message:"fatal: Could not read from remote repository." - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") diff --git a/queries/1695970.yaml b/queries/1695970.yaml index d0848435..b279fde8 100644 --- a/queries/1695970.yaml +++ b/queries/1695970.yaml @@ -1,3 +1,3 @@ query: >- message:"mount: mounting /dev/sr0 on /mnt failed: Device or resource busy" - AND tags:"console.html" + AND (tags:"console.html" OR tags:"job-output.txt") diff --git a/queries/1718990.yaml b/queries/1718990.yaml index 2fbcb1f0..95f8ff56 100644 --- a/queries/1718990.yaml +++ b/queries/1718990.yaml @@ -1,3 +1,3 @@ query: > message:"HTTP 500 curl 22 The requested URL returned error: 500 Internal Server Error" AND - filename:"console.html" + (filename:"console.html" OR filename:"job-output.txt")