From 5193d0010e01de52b4680826defa31b231bdba08 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Date: Wed, 28 Feb 2018 15:36:12 +0000 Subject: [PATCH] Setup tox.ini for unit and pep8 testing This patch includes fixes to all the pep8 warnings. Change-Id: I4f87e1428258824509d2d83879312a1f0dff0a8a --- .pylintrc | 127 +++++++++++++++++++++++++++++++++ .zuul.yaml | 21 ++++++ oslogmerger/oslogmerger.py | 37 +++++----- oslogmerger/probes/netprobe.py | 3 +- requirements.txt | 2 + setup.cfg | 36 +++++++++- setup.py | 69 +----------------- test-requirements.txt | 11 +++ tools/coding-checks.sh | 58 +++++++++++++++ tox.ini | 59 +++++++++++++++ 10 files changed, 335 insertions(+), 88 deletions(-) create mode 100644 .pylintrc create mode 100644 .zuul.yaml create mode 100644 requirements.txt create mode 100644 test-requirements.txt create mode 100755 tools/coding-checks.sh create mode 100644 tox.ini diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..3a76fd6 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,127 @@ +# The format of this file isn't really documented; just use --generate-rcfile +[MASTER] +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=.git,tests + +[MESSAGES CONTROL] +# NOTE(gus): This is a long list. A number of these are important and +# should be re-enabled once the offending code is fixed (or marked +# with a local disable) +disable= +# "F" Fatal errors that prevent further processing + import-error, +# "I" Informational noise + locally-disabled, +# "E" Error for important programming issues (likely bugs) + access-member-before-definition, + anomalous-backslash-in-string, + no-member, + no-method-argument, + no-self-argument, + not-an-iterable, +# "W" Warnings for stylistic problems or minor programming issues + abstract-method, + arguments-differ, + attribute-defined-outside-init, + bad-builtin, + bad-indentation, + broad-except, + dangerous-default-value, + deprecated-lambda, + expression-not-assigned, + fixme, + global-statement, + literal-comparison, + no-init, + non-parent-init-called, + not-callable, + protected-access, + redefined-builtin, + redefined-outer-name, + signature-differs, + star-args, + super-init-not-called, + super-on-old-class, + unpacking-non-sequence, + unused-argument, + unused-import, + unused-variable, + useless-super-delegation, + nonstandard-exception, +# "C" Coding convention violations + bad-continuation, + consider-iterating-dictionary, + consider-using-enumerate, + invalid-name, + len-as-condition, + misplaced-comparison-constant, + missing-docstring, + singleton-comparison, + superfluous-parens, + ungrouped-imports, + wrong-import-order, +# "R" Refactor recommendations + abstract-class-little-used, + abstract-class-not-used, + consider-merging-isinstance, + consider-using-ternary, + duplicate-code, + interface-not-implemented, + no-else-return, + no-self-use, + redefined-argument-from-local, + simplifiable-if-statement, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements + +[BASIC] +# Variable names can be 1 to 31 characters long, with lowercase and underscores +variable-rgx=[a-z_][a-z0-9_]{0,30}$ + +# Argument names can be 2 to 31 characters long, with lowercase and underscores +argument-rgx=[a-z_][a-z0-9_]{1,30}$ + +# Method names should be at least 3 characters long +# and be lowercased with underscores +method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$ + +# Don't require docstrings on tests. +no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=79 + +[VARIABLES] +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +# _ is used by our localization +additional-builtins=_ + +[CLASSES] +# List of interface methods to ignore, separated by a comma. +ignore-iface-methods= + +[IMPORTS] +# Deprecated modules which should not be used, separated by a comma +deprecated-modules= +# should use oslo_serialization.jsonutils + json + +[TYPECHECK] +# List of module names for which member attributes should not be checked +ignored-modules=six.moves,_MovedItems + +[REPORTS] +# Tells whether to display a full report or only the messages +reports=no diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..16dd3e6 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,21 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + check: + jobs: + - tox-pep8 + gate: + jobs: + - tox-pep8 diff --git a/oslogmerger/oslogmerger.py b/oslogmerger/oslogmerger.py index c57750c..1344e62 100644 --- a/oslogmerger/oslogmerger.py +++ b/oslogmerger/oslogmerger.py @@ -1,8 +1,6 @@ from __future__ import print_function import argparse from datetime import datetime, timedelta -import dateutil.parser -import dateutil.tz import hashlib import heapq import itertools @@ -12,9 +10,11 @@ import sys import tempfile import time -from six.moves.urllib.parse import urlparse +import dateutil.parser +import dateutil.tz +from six.moves.urllib.request import urlopen -__version__ = '1.1.0' +__version__ = '1.1.1' EXTRALINES_PADDING = " " * 40 CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir() @@ -62,7 +62,6 @@ FILE_MAP = { 'lbaas-agent': 'LBAAS', 'metadata-agent': 'META', 'metering-agent': 'MTR', - 'openvswitch-agent': 'VSWI', 'server': 'API', 'linuxbridge-agent': 'SVC', 'netprobe': 'NET', @@ -77,7 +76,6 @@ FILE_MAP = { 'ceilometer-dbsync': 'DBSY', 'central': 'CENT', 'collector': 'COLL', - 'compute': 'CPT', } @@ -175,14 +173,14 @@ class LibvirtdParser(LogParser): This parser handles libvirtd.log and libvirt domain logs. Domain logs contain a mixture of libvirt and qemu logs, hence the 2 log formats. """ - LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date - '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time - '(' # - '([+-])(\d{2})(\d{2})' # Timezone - '):\s*') # + LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date + '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time + '(' + '([+-])(\d{2})(\d{2})' # Timezone + '):\s*') - QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date - '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time + QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date + '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time def parse_line(self, line): m = self.LIBVIRT.match(line) @@ -248,11 +246,11 @@ class RawSyslog(LogParser): # manually anyway. HEADER = re.compile('<\d+>\d+\s' '(' - '(\d{4})-(\d{2})-(\d{2})T' # Date - '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time - '(' # - '([+-])(\d{2}):(\d{2})' # Timezone - ')' # + '(\d{4})-(\d{2})-(\d{2})T' # Date + '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time + '(' + '([+-])(\d{2}):(\d{2})' # Timezone + ')' ')\s*') def parse_line(self, line): @@ -324,7 +322,7 @@ class TSLogParser(LogParser): def parse_line(self, line): end, timestamp = self._read_timestamp(line) dt = self.start_date + timedelta(seconds=timestamp) - dt = dt.replace(tzinfo = self.cfg.default_tz) + dt = dt.replace(tzinfo=self.cfg.default_tz) return dt, line[:end + 1], line[end + 1:] @@ -453,7 +451,6 @@ DETECTED_LOG_TYPES = [ def process_logs(cfg): - filename_alias = {} logs = [] paths_aliases = {} diff --git a/oslogmerger/probes/netprobe.py b/oslogmerger/probes/netprobe.py index 0a40138..f1f1305 100644 --- a/oslogmerger/probes/netprobe.py +++ b/oslogmerger/probes/netprobe.py @@ -88,7 +88,7 @@ def spawn_tcpdump(interface, netns=None, reads, writes, excs = select.select([tcpdump.stdout], [], []) try: out = reads[0].readline() - except: + except Exception: continue if out == '': break @@ -187,5 +187,6 @@ def main(): output = open(args.output_file, 'w', 0) scan_loop(args) + if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d87e947 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +python-dateutil +six diff --git a/setup.cfg b/setup.cfg index af57bc0..876a791 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1 +1,35 @@ -[bdist_wheel] +[metadata] +name = os-log-merger +summary = OpenStack Log merge tool +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = https://github.com/openstack/os-log-merger +classifier = + Environment :: OpenStack + Intended Audience :: Developers + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Topic:: Utilities +keywords = OpenStack log merger +license = Apache Software License + +[files] +packages = + oslogmerger + +[entry_points] +console_scripts = + os-log-merger=oslogmerger.oslogmerger:main + oslogmerger=oslogmerger.oslogmerger:main + netprobe=oslogmerger.probes.netprobe:main + +[wheel] +universal = 1 diff --git a/setup.py b/setup.py index 1ca38f9..9c697b9 100644 --- a/setup.py +++ b/setup.py @@ -1,71 +1,8 @@ -from setuptools import setup, find_packages -from codecs import open -from os import path +from setuptools import setup from oslogmerger.oslogmerger import __version__ -here = path.abspath(path.dirname(__file__)) - -# Get the long description from the README file -with open(path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() - setup( - name='os-log-merger', - version=__version__, - description='OpenStack Log merge tool', - long_description=long_description, - - # The project's main homepage. - url='https://github.com/mangelajo/os-log-merger', - - # Author details - author='Miguel Angel Ajo, Gorka Eguileor', - author_email='majopela@redhat.com, geguileo@redhat.com', - - # Choose your license - license='Apache Software License', - - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', - - # Indicate who your project is intended for - 'Intended Audience :: Developers', - - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: Apache Software License', - 'Topic :: Utilities', - 'Environment :: OpenStack', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.7', - ], - - # What does your project relate to? - keywords='OpenStack log merger', - - # You can just specify the packages manually here if your project is - # simple. Or you can use find_packages(). - packages=find_packages(exclude=['contrib', 'docs', 'tests']), - - py_modules=['oslogmerger'], - install_requires=[], - # extras_require={ - # 'dev': ['check-manifest'], - # 'test': ['coverage'], - # }, - - entry_points={ - 'console_scripts': [ - 'os-log-merger=oslogmerger.oslogmerger:main', - 'oslogmerger=oslogmerger.oslogmerger:main', - 'netprobe=oslogmerger.probes.netprobe:main', - ], - }, + setup_requires=['pbr>=2.0.0'], + pbr=True, ) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..9154faf --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,11 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +flake8 +flake8-import-order==0.12 # LGPLv3 +mock>=2.0.0 # BSD +python-subunit>=0.0.18 # Apache-2.0/BSD +testtools>=1.4.0 # MIT +oslotest>=1.10.0 # Apache-2.0 +os-testr>=1.0.0 # Apache-2.0 +pylint==1.4.5 # GPLv2 diff --git a/tools/coding-checks.sh b/tools/coding-checks.sh new file mode 100755 index 0000000..0985ec8 --- /dev/null +++ b/tools/coding-checks.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +set -eu + +usage () { + echo "Usage: $0 [OPTION]..." + echo "Run os-log-mergers's coding check(s)" + echo "" + echo " -Y, --pylint [] Run pylint check on the entire oslogmerger module or just files changed in basecommit (e.g. HEAD~1)" + echo " -h, --help Print this usage message" + echo + exit 0 +} + +process_options () { + i=1 + while [ $i -le $# ]; do + eval opt=\$$i + case $opt in + -h|--help) usage;; + -Y|--pylint) pylint=1;; + *) scriptargs="$scriptargs $opt" + esac + i=$((i+1)) + done +} + +run_pylint () { + local target="${scriptargs:-all}" + + if [ "$target" = "all" ]; then + files="oslogmerger" + else + case "$target" in + *HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");; + *) echo "$target is an unrecognized basecommit"; exit 1;; + esac + fi + + echo "Running pylint..." + echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..." + if [ -n "${files}" ]; then + pylint --rcfile=.pylintrc --output-format=colorized ${files} + else + echo "No python changes in this commit, pylint check not required." + exit 0 + fi +} + +scriptargs= +pylint=1 + +process_options $@ + +if [ $pylint -eq 1 ]; then + run_pylint + exit 0 +fi diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0fdbad7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,59 @@ +[tox] +envlist = py35,pep8 +minversion = 2.3.2 +skipsdist = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONWARNINGS=default::DeprecationWarning +passenv = TRACE_FAILONLY GENERATE_HASHES http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +usedevelop = True +install_command = + pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +whitelist_externals = sh +commands = + {toxinidir}/tools/ostestr_compat_shim.sh {posargs} +# there is also secret magic in ostestr which lets you run in a fail only +# mode. To do this define the TRACE_FAILONLY environmental variable. + +[testenv:common] +# Fake job to define environment variables shared between dsvm/non-dsvm jobs +setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180} +commands = false + +[testenv:pep8] +basepython = python2.7 +deps = + {[testenv]deps} +commands= + flake8 + sh ./tools/coding-checks.sh --pylint '{posargs}' +whitelist_externals = + sh + bash + +[testenv:venv] +commands = {posargs} + +[flake8] +# E125 continuation line does not distinguish itself from next logical line +# E126 continuation line over-indented for hanging indent +# E128 continuation line under-indented for visual indent +# E129 visually indented line with same indent as next logical line +# E265 block comment should start with '# ' +# H404 multi line docstring should start with a summary +# H405 multi line docstring summary not separated with an empty line +# N530 direct neutron imports not allowed +# TODO(ihrachys) figure out what to do with N534 and N536 +# N534 Untranslated exception message +# N536 Use assertIsNone rather than assertEqual to check for None values +ignore = E125,E126,E128,E129,E265,H404,H405,N530,N534,N536 +# H106: Don't put vim configuration in source files +# H203: Use assertIs(Not)None to check for None +# H904: Delay string interpolations at logging calls +enable-extensions=H106,H203,H904 +show-source = true +exclude = ./.*,build,dist,doc +import-order-style = pep8