Setup tox.ini for unit and pep8 testing

This patch includes fixes to all the pep8 warnings.

Change-Id: I4f87e1428258824509d2d83879312a1f0dff0a8a
This commit is contained in:
Miguel Angel Ajo 2018-02-28 15:36:12 +00:00
parent 809bb6add3
commit 5193d0010e
10 changed files with 335 additions and 88 deletions

127
.pylintrc Normal file
View File

@ -0,0 +1,127 @@
# The format of this file isn't really documented; just use --generate-rcfile
[MASTER]
# Add <file or directory> to the black list. It should be a base name, not a
# path. You may set this option multiple times.
ignore=.git,tests
[MESSAGES CONTROL]
# NOTE(gus): This is a long list. A number of these are important and
# should be re-enabled once the offending code is fixed (or marked
# with a local disable)
disable=
# "F" Fatal errors that prevent further processing
import-error,
# "I" Informational noise
locally-disabled,
# "E" Error for important programming issues (likely bugs)
access-member-before-definition,
anomalous-backslash-in-string,
no-member,
no-method-argument,
no-self-argument,
not-an-iterable,
# "W" Warnings for stylistic problems or minor programming issues
abstract-method,
arguments-differ,
attribute-defined-outside-init,
bad-builtin,
bad-indentation,
broad-except,
dangerous-default-value,
deprecated-lambda,
expression-not-assigned,
fixme,
global-statement,
literal-comparison,
no-init,
non-parent-init-called,
not-callable,
protected-access,
redefined-builtin,
redefined-outer-name,
signature-differs,
star-args,
super-init-not-called,
super-on-old-class,
unpacking-non-sequence,
unused-argument,
unused-import,
unused-variable,
useless-super-delegation,
nonstandard-exception,
# "C" Coding convention violations
bad-continuation,
consider-iterating-dictionary,
consider-using-enumerate,
invalid-name,
len-as-condition,
misplaced-comparison-constant,
missing-docstring,
singleton-comparison,
superfluous-parens,
ungrouped-imports,
wrong-import-order,
# "R" Refactor recommendations
abstract-class-little-used,
abstract-class-not-used,
consider-merging-isinstance,
consider-using-ternary,
duplicate-code,
interface-not-implemented,
no-else-return,
no-self-use,
redefined-argument-from-local,
simplifiable-if-statement,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-instance-attributes,
too-many-lines,
too-many-locals,
too-many-nested-blocks,
too-many-public-methods,
too-many-return-statements,
too-many-statements
[BASIC]
# Variable names can be 1 to 31 characters long, with lowercase and underscores
variable-rgx=[a-z_][a-z0-9_]{0,30}$
# Argument names can be 2 to 31 characters long, with lowercase and underscores
argument-rgx=[a-z_][a-z0-9_]{1,30}$
# Method names should be at least 3 characters long
# and be lowercased with underscores
method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
# Don't require docstrings on tests.
no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=79
[VARIABLES]
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
# _ is used by our localization
additional-builtins=_
[CLASSES]
# List of interface methods to ignore, separated by a comma.
ignore-iface-methods=
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=
# should use oslo_serialization.jsonutils
json
[TYPECHECK]
# List of module names for which member attributes should not be checked
ignored-modules=six.moves,_MovedItems
[REPORTS]
# Tells whether to display a full report or only the messages
reports=no

21
.zuul.yaml Normal file
View File

@ -0,0 +1,21 @@
# Copyright 2018 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- project:
check:
jobs:
- tox-pep8
gate:
jobs:
- tox-pep8

View File

@ -1,8 +1,6 @@
from __future__ import print_function
import argparse
from datetime import datetime, timedelta
import dateutil.parser
import dateutil.tz
import hashlib
import heapq
import itertools
@ -12,9 +10,11 @@ import sys
import tempfile
import time
from six.moves.urllib.parse import urlparse
import dateutil.parser
import dateutil.tz
from six.moves.urllib.request import urlopen
__version__ = '1.1.0'
__version__ = '1.1.1'
EXTRALINES_PADDING = " " * 40
CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir()
@ -62,7 +62,6 @@ FILE_MAP = {
'lbaas-agent': 'LBAAS',
'metadata-agent': 'META',
'metering-agent': 'MTR',
'openvswitch-agent': 'VSWI',
'server': 'API',
'linuxbridge-agent': 'SVC',
'netprobe': 'NET',
@ -77,7 +76,6 @@ FILE_MAP = {
'ceilometer-dbsync': 'DBSY',
'central': 'CENT',
'collector': 'COLL',
'compute': 'CPT',
}
@ -175,14 +173,14 @@ class LibvirtdParser(LogParser):
This parser handles libvirtd.log and libvirt domain logs. Domain logs
contain a mixture of libvirt and qemu logs, hence the 2 log formats.
"""
LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time
'(' #
'([+-])(\d{2})(\d{2})' # Timezone
'):\s*') #
LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time
'('
'([+-])(\d{2})(\d{2})' # Timezone
'):\s*')
QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time
QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time
def parse_line(self, line):
m = self.LIBVIRT.match(line)
@ -248,11 +246,11 @@ class RawSyslog(LogParser):
# manually anyway.
HEADER = re.compile('<\d+>\d+\s'
'('
'(\d{4})-(\d{2})-(\d{2})T' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time
'(' #
'([+-])(\d{2}):(\d{2})' # Timezone
')' #
'(\d{4})-(\d{2})-(\d{2})T' # Date
'(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time
'('
'([+-])(\d{2}):(\d{2})' # Timezone
')'
')\s*')
def parse_line(self, line):
@ -324,7 +322,7 @@ class TSLogParser(LogParser):
def parse_line(self, line):
end, timestamp = self._read_timestamp(line)
dt = self.start_date + timedelta(seconds=timestamp)
dt = dt.replace(tzinfo = self.cfg.default_tz)
dt = dt.replace(tzinfo=self.cfg.default_tz)
return dt, line[:end + 1], line[end + 1:]
@ -453,7 +451,6 @@ DETECTED_LOG_TYPES = [
def process_logs(cfg):
filename_alias = {}
logs = []
paths_aliases = {}

View File

@ -88,7 +88,7 @@ def spawn_tcpdump(interface, netns=None,
reads, writes, excs = select.select([tcpdump.stdout], [], [])
try:
out = reads[0].readline()
except:
except Exception:
continue
if out == '':
break
@ -187,5 +187,6 @@ def main():
output = open(args.output_file, 'w', 0)
scan_loop(args)
if __name__ == '__main__':
main()

2
requirements.txt Normal file
View File

@ -0,0 +1,2 @@
python-dateutil
six

View File

@ -1 +1,35 @@
[bdist_wheel]
[metadata]
name = os-log-merger
summary = OpenStack Log merge tool
description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = https://github.com/openstack/os-log-merger
classifier =
Environment :: OpenStack
Intended Audience :: Developers
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Topic:: Utilities
keywords = OpenStack log merger
license = Apache Software License
[files]
packages =
oslogmerger
[entry_points]
console_scripts =
os-log-merger=oslogmerger.oslogmerger:main
oslogmerger=oslogmerger.oslogmerger:main
netprobe=oslogmerger.probes.netprobe:main
[wheel]
universal = 1

View File

@ -1,71 +1,8 @@
from setuptools import setup, find_packages
from codecs import open
from os import path
from setuptools import setup
from oslogmerger.oslogmerger import __version__
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
setup(
name='os-log-merger',
version=__version__,
description='OpenStack Log merge tool',
long_description=long_description,
# The project's main homepage.
url='https://github.com/mangelajo/os-log-merger',
# Author details
author='Miguel Angel Ajo, Gorka Eguileor',
author_email='majopela@redhat.com, geguileo@redhat.com',
# Choose your license
license='Apache Software License',
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 3 - Alpha',
# Indicate who your project is intended for
'Intended Audience :: Developers',
# Pick your license as you wish (should match "license" above)
'License :: OSI Approved :: Apache Software License',
'Topic :: Utilities',
'Environment :: OpenStack',
# Specify the Python versions you support here. In particular, ensure
# that you indicate whether you support Python 2, Python 3 or both.
'Programming Language :: Python :: 2.7',
],
# What does your project relate to?
keywords='OpenStack log merger',
# You can just specify the packages manually here if your project is
# simple. Or you can use find_packages().
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
py_modules=['oslogmerger'],
install_requires=[],
# extras_require={
# 'dev': ['check-manifest'],
# 'test': ['coverage'],
# },
entry_points={
'console_scripts': [
'os-log-merger=oslogmerger.oslogmerger:main',
'oslogmerger=oslogmerger.oslogmerger:main',
'netprobe=oslogmerger.probes.netprobe:main',
],
},
setup_requires=['pbr>=2.0.0'],
pbr=True,
)

11
test-requirements.txt Normal file
View File

@ -0,0 +1,11 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
flake8
flake8-import-order==0.12 # LGPLv3
mock>=2.0.0 # BSD
python-subunit>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
oslotest>=1.10.0 # Apache-2.0
os-testr>=1.0.0 # Apache-2.0
pylint==1.4.5 # GPLv2

58
tools/coding-checks.sh Executable file
View File

@ -0,0 +1,58 @@
#!/bin/sh
set -eu
usage () {
echo "Usage: $0 [OPTION]..."
echo "Run os-log-mergers's coding check(s)"
echo ""
echo " -Y, --pylint [<basecommit>] Run pylint check on the entire oslogmerger module or just files changed in basecommit (e.g. HEAD~1)"
echo " -h, --help Print this usage message"
echo
exit 0
}
process_options () {
i=1
while [ $i -le $# ]; do
eval opt=\$$i
case $opt in
-h|--help) usage;;
-Y|--pylint) pylint=1;;
*) scriptargs="$scriptargs $opt"
esac
i=$((i+1))
done
}
run_pylint () {
local target="${scriptargs:-all}"
if [ "$target" = "all" ]; then
files="oslogmerger"
else
case "$target" in
*HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");;
*) echo "$target is an unrecognized basecommit"; exit 1;;
esac
fi
echo "Running pylint..."
echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..."
if [ -n "${files}" ]; then
pylint --rcfile=.pylintrc --output-format=colorized ${files}
else
echo "No python changes in this commit, pylint check not required."
exit 0
fi
}
scriptargs=
pylint=1
process_options $@
if [ $pylint -eq 1 ]; then
run_pylint
exit 0
fi

59
tox.ini Normal file
View File

@ -0,0 +1,59 @@
[tox]
envlist = py35,pep8
minversion = 2.3.2
skipsdist = True
[testenv]
setenv = VIRTUAL_ENV={envdir}
PYTHONWARNINGS=default::DeprecationWarning
passenv = TRACE_FAILONLY GENERATE_HASHES http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
usedevelop = True
install_command =
pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals = sh
commands =
{toxinidir}/tools/ostestr_compat_shim.sh {posargs}
# there is also secret magic in ostestr which lets you run in a fail only
# mode. To do this define the TRACE_FAILONLY environmental variable.
[testenv:common]
# Fake job to define environment variables shared between dsvm/non-dsvm jobs
setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
commands = false
[testenv:pep8]
basepython = python2.7
deps =
{[testenv]deps}
commands=
flake8
sh ./tools/coding-checks.sh --pylint '{posargs}'
whitelist_externals =
sh
bash
[testenv:venv]
commands = {posargs}
[flake8]
# E125 continuation line does not distinguish itself from next logical line
# E126 continuation line over-indented for hanging indent
# E128 continuation line under-indented for visual indent
# E129 visually indented line with same indent as next logical line
# E265 block comment should start with '# '
# H404 multi line docstring should start with a summary
# H405 multi line docstring summary not separated with an empty line
# N530 direct neutron imports not allowed
# TODO(ihrachys) figure out what to do with N534 and N536
# N534 Untranslated exception message
# N536 Use assertIsNone rather than assertEqual to check for None values
ignore = E125,E126,E128,E129,E265,H404,H405,N530,N534,N536
# H106: Don't put vim configuration in source files
# H203: Use assertIs(Not)None to check for None
# H904: Delay string interpolations at logging calls
enable-extensions=H106,H203,H904
show-source = true
exclude = ./.*,build,dist,doc
import-order-style = pep8