From 539d13e81610cbc70c948c74f1d064175ce55384 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Sat, 27 Oct 2018 10:20:28 +0100 Subject: [PATCH] Fix flake8 failures and make its execution more consistent - upgrade flake8 to 3.6.0 - remove use of hacking since pre-commit replaced it - mentioned minimal flake8 version on pre-commit config - fixed newer linting errors - disable W504 as it seems not possible to fix (conflict with W403) - replaced buggy tox deps installation with native pip install, which address the issue of missing to install new deps when the reqs files are updated, developer being forced to to recreate the virtualenv. - prepare for migration from tox-pep8 to tox-linters, now these being just aliases. Change-Id: I310578dce215aaf00a5b2d54716f90da9a1ecb4d Depends-On: https://review.openstack.org/#/c/613726/ --- .pre-commit-config.yaml | 5 ++++- jenkins_jobs/alphanum.py | 6 +++--- jenkins_jobs/cache.py | 2 +- jenkins_jobs/formatter.py | 2 +- jenkins_jobs/modules/builders.py | 18 +++++++++--------- jenkins_jobs/modules/project_multibranch.py | 6 +++--- jenkins_jobs/modules/publishers.py | 10 +++++----- jenkins_jobs/modules/scm.py | 6 +++--- jenkins_jobs/sphinx/yaml.py | 2 +- setup.cfg | 4 +++- test-requirements.txt | 3 +-- tox.ini | 21 ++++++++++++++++----- 12 files changed, 50 insertions(+), 35 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff6899acf..9d14a1433 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,10 +2,13 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v1.2.1-1 + rev: v2.0.0 hooks: - id: check-added-large-files - id: end-of-file-fixer - id: flake8 args: ['--config=setup.cfg'] + language: python + types: [python] + additional_dependencies: [flake8>=3.6.0] - id: trailing-whitespace diff --git a/jenkins_jobs/alphanum.py b/jenkins_jobs/alphanum.py index 481a0a93b..7eb44bfc1 100755 --- a/jenkins_jobs/alphanum.py +++ b/jenkins_jobs/alphanum.py @@ -32,9 +32,9 @@ sorted( foo, key=AlphanumSort) import re -re_chunk = re.compile("([\D]+|[\d]+)") -re_letters = re.compile("\D+") -re_numbers = re.compile("\d+") +re_chunk = re.compile(r"([\D]+|[\d]+)") +re_letters = re.compile(r"\D+") +re_numbers = re.compile(r"\d+") def getchunk(item): diff --git a/jenkins_jobs/cache.py b/jenkins_jobs/cache.py index 61a79e574..d64d6d9fb 100644 --- a/jenkins_jobs/cache.py +++ b/jenkins_jobs/cache.py @@ -43,7 +43,7 @@ class JobCache(object): def __init__(self, jenkins_url, flush=False): cache_dir = self.get_cache_dir() # One cache per remote Jenkins URL: - host_vary = re.sub('[^A-Za-z0-9\-\~]', '_', jenkins_url) + host_vary = re.sub(r'[^A-Za-z0-9\-\~]', '_', jenkins_url) self.cachefilename = os.path.join( cache_dir, 'cache-host-jobs-' + host_vary + '.yml') diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index 9b1d0647e..bd2576b9e 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -81,7 +81,7 @@ class CustomFormatter(Formatter): Custom formatter to allow non-existing key references when formatting a string """ - _expr = """ + _expr = r""" (?\w+) # key diff --git a/jenkins_jobs/modules/builders.py b/jenkins_jobs/modules/builders.py index d116cf074..9a4e18389 100644 --- a/jenkins_jobs/modules/builders.py +++ b/jenkins_jobs/modules/builders.py @@ -2888,7 +2888,7 @@ def cmake(registry, xml_parent, data): def dsl(registry, xml_parent, data): - """yaml: dsl + r"""yaml: dsl Process Job DSL Requires the Jenkins :jenkins-wiki:`Job DSL plugin `. @@ -3475,7 +3475,7 @@ def jms_messaging(registry, xml_parent, data): def openshift_build_verify(registry, xml_parent, data): - """yaml: openshift-build-verify + r"""yaml: openshift-build-verify Performs the equivalent of an 'oc get builds` command invocation for the provided buildConfig key provided; once the list of builds are obtained, the state of the latest build is inspected for up to a minute to see if @@ -3524,7 +3524,7 @@ def openshift_build_verify(registry, xml_parent, data): def openshift_builder(registry, xml_parent, data): - """yaml: openshift-builder + r"""yaml: openshift-builder Perform builds in OpenShift for the job. Requires the Jenkins :jenkins-wiki:`OpenShift Pipeline Plugin `. @@ -3580,7 +3580,7 @@ def openshift_builder(registry, xml_parent, data): def openshift_creator(registry, xml_parent, data): - """yaml: openshift-creator + r"""yaml: openshift-creator Performs the equivalent of an oc create command invocation; this build step takes in the provided JSON or YAML text, and if it conforms to OpenShift schema, creates whichever @@ -3628,7 +3628,7 @@ def openshift_creator(registry, xml_parent, data): def openshift_dep_verify(registry, xml_parent, data): - """yaml: openshift-dep-verify + r"""yaml: openshift-dep-verify Determines whether the expected set of DeploymentConfig's, ReplicationController's, and active replicas are present based on prior use of the scaler (2) and deployer (3) steps @@ -3679,7 +3679,7 @@ def openshift_dep_verify(registry, xml_parent, data): def openshift_deployer(registry, xml_parent, data): - """yaml: openshift-deployer + r"""yaml: openshift-deployer Start a deployment in OpenShift for the job. Requires the Jenkins :jenkins-wiki:`OpenShift Pipeline Plugin `. @@ -3725,7 +3725,7 @@ def openshift_deployer(registry, xml_parent, data): def openshift_img_tagger(registry, xml_parent, data): - """yaml: openshift-img-tagger + r"""yaml: openshift-img-tagger Performs the equivalent of an oc tag command invocation in order to manipulate tags for images in OpenShift ImageStream's Requires the Jenkins :jenkins-wiki:`OpenShift @@ -3776,7 +3776,7 @@ def openshift_img_tagger(registry, xml_parent, data): def openshift_scaler(registry, xml_parent, data): - """yaml: openshift-scaler + r"""yaml: openshift-scaler Scale deployments in OpenShift for the job. Requires the Jenkins :jenkins-wiki:`OpenShift Pipeline Plugin `. @@ -3823,7 +3823,7 @@ def openshift_scaler(registry, xml_parent, data): def openshift_svc_verify(registry, xml_parent, data): - """yaml: openshift-svc-verify + r"""yaml: openshift-svc-verify Verify a service is up in OpenShift for the job. Requires the Jenkins :jenkins-wiki:`OpenShift Pipeline Plugin `. diff --git a/jenkins_jobs/modules/project_multibranch.py b/jenkins_jobs/modules/project_multibranch.py index bfe460e9b..3be0cf69b 100644 --- a/jenkins_jobs/modules/project_multibranch.py +++ b/jenkins_jobs/modules/project_multibranch.py @@ -290,7 +290,7 @@ class WorkflowMultiBranchDefaults(WorkflowMultiBranch): def bitbucket_scm(xml_parent, data): - """Configure BitBucket scm + r"""Configure BitBucket scm Requires the :jenkins-wiki:`Bitbucket Branch Source Plugin `. @@ -626,7 +626,7 @@ def gerrit_scm(xml_parent, data): def git_scm(xml_parent, data): - """Configure Git SCM + r"""Configure Git SCM Requires the :jenkins-wiki:`Git Plugin `. @@ -738,7 +738,7 @@ def git_scm(xml_parent, data): def github_scm(xml_parent, data): - """Configure GitHub SCM + r"""Configure GitHub SCM Requires the :jenkins-wiki:`GitHub Branch Source Plugin `. diff --git a/jenkins_jobs/modules/publishers.py b/jenkins_jobs/modules/publishers.py index d300a8da2..70031f57b 100644 --- a/jenkins_jobs/modules/publishers.py +++ b/jenkins_jobs/modules/publishers.py @@ -1658,7 +1658,7 @@ def violations(registry, xml_parent, data): def findbugs(registry, xml_parent, data): - """yaml: findbugs + r"""yaml: findbugs FindBugs reporting for builds Requires the Jenkins :jenkins-wiki:`FindBugs Plugin @@ -2962,7 +2962,7 @@ def sounds(parser, xml_parent, data): def performance(registry, xml_parent, data): - """yaml: performance + r"""yaml: performance Publish performance test results from jmeter and junit. Requires the Jenkins :jenkins-wiki:`Performance Plugin `. @@ -4391,7 +4391,7 @@ def warnings(registry, xml_parent, data): def sloccount(registry, xml_parent, data): - """yaml: sloccount + r"""yaml: sloccount Generates the trend report for SLOCCount Requires the Jenkins :jenkins-wiki:`SLOCCount Plugin `. @@ -7220,7 +7220,7 @@ def jms_messaging(registry, xml_parent, data): def openshift_build_canceller(registry, xml_parent, data): - """yaml: openshift-build-canceller + r"""yaml: openshift-build-canceller This action is intended to provide cleanup for a Jenkins job which failed because a build is hung (instead of terminating with a failure code); this step will allow you to perform the equivalent of a oc cancel-build @@ -7271,7 +7271,7 @@ def openshift_build_canceller(registry, xml_parent, data): def openshift_deploy_canceller(registry, xml_parent, data): - """yaml: openshift-deploy-canceller + r"""yaml: openshift-deploy-canceller This action is intended to provide cleanup for any OpenShift deployments left running when the Job completes; this step will allow you to perform the equivalent of a oc deploy --cancel for the provided deployment config. diff --git a/jenkins_jobs/modules/scm.py b/jenkins_jobs/modules/scm.py index 180c50923..519550ced 100644 --- a/jenkins_jobs/modules/scm.py +++ b/jenkins_jobs/modules/scm.py @@ -46,7 +46,7 @@ import jenkins_jobs.modules.helpers as helpers def git(registry, xml_parent, data): - """yaml: git + r"""yaml: git Specifies the git SCM repository for this job. Requires the Jenkins :jenkins-wiki:`Git Plugin `. @@ -994,7 +994,7 @@ def svn(registry, xml_parent, data): def tfs(registry, xml_parent, data): - """yaml: tfs + r"""yaml: tfs Specifies the Team Foundation Server repository for this job. Requires the Jenkins :jenkins-wiki:`Team Foundation Server Plugin `. @@ -1216,7 +1216,7 @@ def hg(self, xml_parent, data): def openshift_img_streams(registry, xml_parent, data): - """yaml: openshift-img-streams + r"""yaml: openshift-img-streams Rather than a Build step extension plugin, this is an extension of the Jenkins SCM plugin, where this baked-in polling mechanism provided by Jenkins is leveraged by exposing some of the common semantics between diff --git a/jenkins_jobs/sphinx/yaml.py b/jenkins_jobs/sphinx/yaml.py index d12fa580a..2c7083f75 100644 --- a/jenkins_jobs/sphinx/yaml.py +++ b/jenkins_jobs/sphinx/yaml.py @@ -29,7 +29,7 @@ from sphinx.ext.autodoc import FunctionDocumenter from sphinx.locale import _ -yaml_sig_re = re.compile('yaml:\s*(.*)') +yaml_sig_re = re.compile(r'yaml:\s*(.*)') class PyYAMLFunction(PyModulelevel): diff --git a/setup.cfg b/setup.cfg index 82caa8876..e642b4281 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,6 +101,8 @@ jenkins_jobs.modules = [flake8] # These are ignored intentionally in openstack-infra projects; please # don't submit patches that solely correct them or enable them. -ignore = E125,E128,H +# W504 is controversial an apparently conflicting with W503, being impossible +# to solve both of them while still keeping the line length limited. +ignore = E125,E128,H,W504 show-source = True exclude = .virtualenv,.venv,.tox,dist,build,*.egg,.test diff --git a/test-requirements.txt b/test-requirements.txt index da8910bec..37767a6f5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,6 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=1.1.0 # Apache-2.0 coverage>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD @@ -13,4 +12,4 @@ stestr>=2.0.0 # Apache-2.0/BSD tox>=2.9.1 # MIT mock>=2.0 # BSD sphinxcontrib-programoutput -pre-commit +pre-commit>=1.12.0 diff --git a/tox.ini b/tox.ini index da193c0ea..8f2a9d80b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,9 @@ [tox] minversion = 2.2 -envlist = docs, pep8, py34, py35, py36, py27, cover +envlist = linters, docs, py34, py35, py36, py27, cover skip_missing_interpreters = true +# custom vars (no meaning to tox) +install_test_deps = pip install -q -r test-requirements.txt [testenv] setenv = @@ -11,11 +13,11 @@ setenv = VIRTUAL_ENV={envdir} usedevelop = True install_command = pip install {opts} {packages} -deps = -r{toxinidir}/test-requirements.txt # cleanup is needed mostly for dev environmnts and assures that if dev # is switching branch, no lefovers will impact execution. # No git commands here because test should run even from a source archive. commands = + {[tox]install_test_deps} - find . -type f -name "*.pyc" -delete - find . -type d -name "__pycache__" -delete # test that we can call jjb using both variants with same results @@ -39,15 +41,22 @@ setenv = {[testenv]setenv} PYTHON=coverage run --source jenkins_jobs --parallel-mode commands = + {[tox]install_test_deps} stestr run {posargs} coverage combine coverage html -d cover coverage xml -o cover/coverage.xml -[testenv:pep8] +[testenv:linters] basepython = python3 commands = - python -m pre_commit run --all + {[tox]install_test_deps} + python -m pre_commit run --all + +# points to linters, kept only for developer convenience +[testenv:pep8] +envdir={toxworkdir}/linters +commands = {[testenv:linters]commands} [testenv:pyflakes] deps = pyflakes @@ -63,7 +72,9 @@ commands = jenkins-jobs test -o .test/old/out/ .test/old/config/ commands = jenkins-jobs test -o .test/new/out/ .test/new/config/ [testenv:docs] -commands = python setup.py build_sphinx {posargs} +commands = + {[tox]install_test_deps} + python setup.py build_sphinx {posargs} [testenv:docs-linkcheck] # If you are behind a proxy, for this test to work you will need to set