diff --git a/.gitignore b/.gitignore index 2c5ed1837..818b8c64d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ murano/tests/functional/engine/config.conf #Autogenerated sample config file etc/murano/murano.conf.sample + +# pylint autogenerated support files +tools/lintstack.head.py +tools/pylint_exceptions diff --git a/test-requirements.txt b/test-requirements.txt index 1807c8372..d5b56b7c1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,6 +16,7 @@ testresources>=0.2.4 testscenarios>=0.4 testtools>=0.9.36,!=1.2.0 unittest2 +pylint>=1.3.0 # GNU GPL v2 # Some of the tests use real MySQL and Postgres databases MySQL-python diff --git a/tools/lintstack.py b/tools/lintstack.py new file mode 100755 index 000000000..419b02e34 --- /dev/null +++ b/tools/lintstack.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# Copyright (c) 2015 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""pylint error checking.""" + +from __future__ import print_function + +import json +import os +import re +import sys + +from pylint import lint +from pylint.reporters import text +from six.moves import cStringIO as StringIO + +enabled_codes="R0801,R0911,R0912,R0913,R0914,R0915" + +KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions" + + +class LintOutput(object): + + _cached_filename = None + _cached_content = None + + def __init__(self, filename, lineno, line_content, code, message, + lintoutput): + self.filename = filename + self.lineno = lineno + self.line_content = line_content + self.code = code + self.message = message + self.lintoutput = lintoutput + + @classmethod + def get_duplicate_code_location(cls, remaining_lines): + module, lineno = remaining_lines.pop(0)[2:].split(":") + filename = module.replace(".", os.sep) + ".py" + return filename, int(lineno) + + @classmethod + def get_line_content(cls, filename, lineno): + if cls._cached_filename != filename: + with open(filename) as f: + cls._cached_content = list(f.readlines()) + cls._cached_filename = filename + # find first non-empty line + lineno -= 1 + while True: + line_content = cls._cached_content[lineno].rstrip() + lineno +=1 + if line_content: + return line_content + + @classmethod + def from_line(cls, line, remaining_lines): + m = re.search(r"(\S+):(\d+): \[(\S+)(, \S*)?] (.*)", line) + if not m: + return None + matched = m.groups() + filename, lineno, code, message = (matched[0], int(matched[1]), + matched[2], matched[-1]) + + # duplicate code output needs special handling + if "duplicate-code" in code: + filename, lineno = cls.get_duplicate_code_location(remaining_lines) + + line_content = cls.get_line_content(filename, lineno) + return cls(filename, lineno, line_content, code, message, + line.rstrip()) + + @classmethod + def from_msg_to_dict(cls, msg): + """From the output of pylint msg, to a dict, where each key + is a unique error identifier, value is a list of LintOutput + """ + result = {} + print(msg) + lines = msg.splitlines() + while lines: + line = lines.pop(0) + obj = cls.from_line(line, lines) + if not obj: + continue + key = obj.key() + if key not in result: + result[key] = [] + result[key].append(obj) + return result + + def key(self): + return self.message, self.line_content.strip() + + def json(self): + return json.dumps(self.__dict__) + + def review_str(self): + return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n" + "%(code)s: %(message)s" % self.__dict__) + + +class ErrorKeys(object): + + @classmethod + def print_json(cls, errors, output=sys.stdout): + print("# automatically generated by tools/lintstack.py", file=output) + for i in sorted(errors.keys()): + print(json.dumps(i), file=output) + + @classmethod + def from_file(cls, filename): + keys = set() + for line in open(filename): + if line and line[0] != "#": + d = json.loads(line) + keys.add(tuple(d)) + return keys + + +def run_pylint(): + buff = StringIO() + reporter = text.ParseableTextReporter(output=buff) + args = ["-rn", "--disable=all", "--enable=" + enabled_codes ,"murano"] + lint.Run(args, reporter=reporter, exit=False) + val = buff.getvalue() + buff.close() + return val + + +def generate_error_keys(msg=None): + print("Generating", KNOWN_PYLINT_EXCEPTIONS_FILE) + if msg is None: + msg = run_pylint() + + print(msg) + errors = LintOutput.from_msg_to_dict(msg) + with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f: + ErrorKeys.print_json(errors, output=f) + + +def validate(newmsg=None): + print("Loading", KNOWN_PYLINT_EXCEPTIONS_FILE) + known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE) + if newmsg is None: + print("Running pylint. Be patient...") + newmsg = run_pylint() + errors = LintOutput.from_msg_to_dict(newmsg) + + print("Unique errors reported by pylint: was %d, now %d." + % (len(known), len(errors))) + passed = True + for err_key, err_list in errors.items(): + for err in err_list: + if err_key not in known: + print(err.lintoutput) + print(err.review_str()) + print() + passed = False + if passed: + print("Congrats! pylint check passed.") + redundant = known - set(errors.keys()) + if redundant: + print("Extra credit: some known pylint exceptions disappeared.") + for i in sorted(redundant): + print(json.dumps(i)) + print("Consider regenerating the exception file if you will.") + else: + print("Please fix the errors above. If you believe they are false " + "positives, run 'tools/lintstack.py generate' to overwrite.") + sys.exit(1) + + +def usage(): + print("""Usage: tools/lintstack.py [generate|validate] + To generate pylint_exceptions file: tools/lintstack.py generate + To validate the current commit: tools/lintstack.py + """) + + +def main(): + option = "validate" + if len(sys.argv) > 1: + option = sys.argv[1] + if option == "generate": + generate_error_keys() + elif option == "validate": + validate() + else: + usage() + + +if __name__ == "__main__": + main() diff --git a/tools/lintstack.sh b/tools/lintstack.sh new file mode 100755 index 000000000..7ba585e17 --- /dev/null +++ b/tools/lintstack.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Use lintstack.py to compare pylint errors. +# We run pylint twice, once on HEAD, once on the code before the latest +# commit for review. +set -e +TOOLS_DIR=$(cd $(dirname "$0") && pwd) +# Get the current branch name. +GITHEAD=`git rev-parse --abbrev-ref HEAD` +if [[ "$GITHEAD" == "HEAD" ]]; then + # In detached head mode, get revision number instead + GITHEAD=`git rev-parse HEAD` + echo "Currently we are at commit $GITHEAD" +else + echo "Currently we are at branch $GITHEAD" +fi + +cp -f $TOOLS_DIR/lintstack.py $TOOLS_DIR/lintstack.head.py + +if git rev-parse HEAD^2 2>/dev/null; then + # The HEAD is a Merge commit. Here, the patch to review is + # HEAD^2, the master branch is at HEAD^1, and the patch was + # written based on HEAD^2~1. + PREV_COMMIT=`git rev-parse HEAD^2~1` + git checkout HEAD~1 + # The git merge is necessary for reviews with a series of patches. + # If not, this is a no-op so won't hurt either. + git merge $PREV_COMMIT +else + # The HEAD is not a merge commit. This won't happen on gerrit. + # Most likely you are running against your own patch locally. + # We assume the patch to examine is HEAD, and we compare it against + # HEAD~1 + git checkout HEAD~1 +fi + +# First generate tools/pylint_exceptions from HEAD~1 +$TOOLS_DIR/lintstack.head.py generate +# Then use that as a reference to compare against HEAD +git checkout $GITHEAD +$TOOLS_DIR/lintstack.head.py +echo "Check passed. FYI: the pylint exceptions are:" +cat $TOOLS_DIR/pylint_exceptions + diff --git a/tox.ini b/tox.ini index 51647d7dd..20cd94064 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,10 @@ commands = deps = flake8 commands = flake8 +[testenv:pylint] +setenv = VIRTUAL_ENV={envdir} +commands = bash tools/lintstack.sh + [testenv:genconfig] commands = oslo-config-generator --config-file etc/oslo-config-generator/murano.conf