From 04e2fcf4d8aa23fb3d727fe021a44324b5b92c33 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Fri, 3 Aug 2018 15:51:57 +0100 Subject: [PATCH] replace pep8 check with generic linters Includes removal of file pattern from pep8 job which prevented it from running on most changes. Linters will use multiple tools and a pattern would make it fail to spot errors. This check should run even if no files are changed, so it would be able to check commit messages. There are no real load implications because linters check is just a simple tox execution which has minimal resource requirements. Includes reactivaction or linting which was not running on the entire repository. This required few minor fixes. Partial-Bug: #1786286 Change-Id: I4d4279309af55c2663e98bd0cdec9224f03c1fa0 --- .../test_write_bash_script.py | 4 ++-- .../emit_releases_file/test_yaml_parsing.py | 2 +- scripts/heat-deploy-times.py | 6 ++++-- scripts/tripleo-jobs-gerrit.py | 21 ++++++++++++------- tox.ini | 13 ++++++++---- zuul.d/layout.yaml | 4 +--- 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/scripts/emit_releases_file/test_write_bash_script.py b/scripts/emit_releases_file/test_write_bash_script.py index e074d5bf2..24569adef 100644 --- a/scripts/emit_releases_file/test_write_bash_script.py +++ b/scripts/emit_releases_file/test_write_bash_script.py @@ -1,8 +1,8 @@ from emit_releases_file import write_releases_dictionary_to_bash -import os import mock from mock import mock_open +import os from six import PY2 import pytest @@ -64,5 +64,5 @@ def test_output_is_sourceable(mock, releases_dictionary): handle = mock() args, _ = handle.write.call_args written_content = args[0] - # TODO: check environment variables + # TODO(Llorente): check environment variables assert (0 == os.system(written_content)) diff --git a/scripts/emit_releases_file/test_yaml_parsing.py b/scripts/emit_releases_file/test_yaml_parsing.py index 571446922..b3f4685dd 100644 --- a/scripts/emit_releases_file/test_yaml_parsing.py +++ b/scripts/emit_releases_file/test_yaml_parsing.py @@ -2,8 +2,8 @@ from emit_releases_file import load_featureset_file import mock import pytest -import yaml from six import PY2 +import yaml if PY2: diff --git a/scripts/heat-deploy-times.py b/scripts/heat-deploy-times.py index b43aecdda..346fafc03 100755 --- a/scripts/heat-deploy-times.py +++ b/scripts/heat-deploy-times.py @@ -21,6 +21,7 @@ import json import sys import time + def process_events(all_events, events): times = {} for event in all_events: @@ -41,12 +42,13 @@ def process_events(all_events, events): elif status == 'CREATE_COMPLETE' or status == 'CREATE_FAILED': times[name]['elapsed'] = etime - times[name]['start'] for name, data in sorted(times.items(), - key = lambda x: x[1]['elapsed'], + key=lambda x: x[1]['elapsed'], reverse=True): elapsed = 'Still in progress' if times[name]['elapsed'] is not None: elapsed = times[name]['elapsed'] - print '%s %s' % (name, elapsed) + print('%s %s' % (name, elapsed)) + if __name__ == '__main__': stdin = sys.stdin.read() diff --git a/scripts/tripleo-jobs-gerrit.py b/scripts/tripleo-jobs-gerrit.py index 420a74780..0c2ed5685 100755 --- a/scripts/tripleo-jobs-gerrit.py +++ b/scripts/tripleo-jobs-gerrit.py @@ -1,11 +1,11 @@ #!/usr/bin/python import argparse -import sys -import subprocess +import datetime import json import re -import datetime +import subprocess +import sys # Do not include the -nv suffix in the job name here. The code will handle # reading both the voting and non-voting forms of the job if they exist. @@ -52,7 +52,7 @@ def get_gerrit_reviews(project, status="open", branch="master", limit="30"): cmd = 'ssh review.openstack.org -p29418 gerrit' \ ' query "%s project: %s branch: %s" --comments' \ ' --format JSON limit: %s --patch-sets --current-patch-set'\ - % (status_query, project, branch,limit) + % (status_query, project, branch, limit) p = subprocess.Popen([cmd], shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = p.stdout @@ -82,7 +82,7 @@ def process_jenkins_comment_message(message, job_names): job_results = {} for line in message.split('\n'): if line and line[0] == '-': - split = line.split(" ",6) + split = line.split(" ", 6) voting_job_name = split[1] if voting_job_name.endswith('-nv'): voting_job_name = voting_job_name[:-3] @@ -108,7 +108,8 @@ def gen_html(data, html_file, table_file, stats_hours, job_names, options): count = 0 reversed_sorted_keys = [(x['id'], x['patchset']) for x in - reversed(sorted(data.values(), key=lambda y: y['ts']))] + reversed(sorted(data.values(), + key=lambda y: y['ts']))] passed_jobs = 0 partial_jobs = 0 failed_jobs = 0 @@ -219,7 +220,10 @@ def main(args=sys.argv[1:]): # project reviews proj_reviews = [] for proj in opts.p.split(","): - proj_reviews.extend(get_gerrit_reviews(proj, status=opts.s, branch=opts.b, limit=opts.l)) + proj_reviews.extend(get_gerrit_reviews(proj, + status=opts.s, + branch=opts.b, + limit=opts.l)) results = {} for review in proj_reviews: for ts, message in get_jenkins_comment_message(review).iteritems(): @@ -242,7 +246,8 @@ def main(args=sys.argv[1:]): results[key].setdefault( 'ci_results', {}).update(ci_results) - gen_html(results, opts.o, "%s-table" % opts.o, 24, job_names,opts) + gen_html(results, opts.o, "%s-table" % opts.o, 24, job_names, opts) + if __name__ == '__main__': exit(main()) diff --git a/tox.ini b/tox.ini index 1f561f20d..0dfcb44b5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pep8, py27, py35 +envlist = linters, py27, py35 [testenv] usedevelop = True @@ -14,10 +14,15 @@ commands = {posargs} deps = pyflakes commands = pyflakes setup.py scripts -[testenv:pep8] -changedir = scripts/emit_releases_file +[testenv:linters] whitelist_externals = bash -commands = flake8 --max-line-length 80 +commands = flake8 --max-line-length 80 {toxinidir} {posargs} + +# deprecated: use linters instead. kept only as a convenience alias +[testenv:pep8] +envdir = {toxworkdir}/linters +whitelist_externals = {[testenv:linters]whitelist_externals} +commands = {[testenv:linters]commands} [testenv:cireport] passenv = diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index 305386d45..575f6ceaf 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -19,9 +19,7 @@ - openstack-tox-py35: files: - ^scripts/emit_releases_file/.*$ - - openstack-tox-pep8: - files: - - ^scripts/emit_releases_file/.*$ + - openstack-tox-linters - tripleo-ci-centos-7-scenario001-multinode-oooq: files: - ^playbooks/tripleo-ci/.*$