summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMiguel Angel Ajo <majopela@redhat.com>2018-02-28 15:36:12 +0000
committerMiguel Angel Ajo <majopela@redhat.com>2018-02-28 23:19:52 +0000
commit5193d0010e01de52b4680826defa31b231bdba08 (patch)
tree649f6da0832d29593a598eca17f9092a224040ef
parent809bb6add302446642ab6e4997ce5a0be8864efe (diff)
Setup tox.ini for unit and pep8 testing
This patch includes fixes to all the pep8 warnings. Change-Id: I4f87e1428258824509d2d83879312a1f0dff0a8a
Notes
Notes (review): Code-Review+2: Miguel Angel Ajo <mangelajo@redhat.com> Workflow+1: Miguel Angel Ajo <mangelajo@redhat.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 28 Feb 2018 23:34:16 +0000 Reviewed-on: https://review.openstack.org/548618 Project: openstack/os-log-merger Branch: refs/heads/master
-rw-r--r--.pylintrc127
-rw-r--r--.zuul.yaml21
-rw-r--r--oslogmerger/oslogmerger.py37
-rw-r--r--oslogmerger/probes/netprobe.py3
-rw-r--r--requirements.txt2
-rw-r--r--setup.cfg36
-rw-r--r--setup.py69
-rw-r--r--test-requirements.txt11
-rwxr-xr-xtools/coding-checks.sh58
-rw-r--r--tox.ini59
10 files changed, 335 insertions, 88 deletions
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 0000000..3a76fd6
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,127 @@
1# The format of this file isn't really documented; just use --generate-rcfile
2[MASTER]
3# Add <file or directory> to the black list. It should be a base name, not a
4# path. You may set this option multiple times.
5ignore=.git,tests
6
7[MESSAGES CONTROL]
8# NOTE(gus): This is a long list. A number of these are important and
9# should be re-enabled once the offending code is fixed (or marked
10# with a local disable)
11disable=
12# "F" Fatal errors that prevent further processing
13 import-error,
14# "I" Informational noise
15 locally-disabled,
16# "E" Error for important programming issues (likely bugs)
17 access-member-before-definition,
18 anomalous-backslash-in-string,
19 no-member,
20 no-method-argument,
21 no-self-argument,
22 not-an-iterable,
23# "W" Warnings for stylistic problems or minor programming issues
24 abstract-method,
25 arguments-differ,
26 attribute-defined-outside-init,
27 bad-builtin,
28 bad-indentation,
29 broad-except,
30 dangerous-default-value,
31 deprecated-lambda,
32 expression-not-assigned,
33 fixme,
34 global-statement,
35 literal-comparison,
36 no-init,
37 non-parent-init-called,
38 not-callable,
39 protected-access,
40 redefined-builtin,
41 redefined-outer-name,
42 signature-differs,
43 star-args,
44 super-init-not-called,
45 super-on-old-class,
46 unpacking-non-sequence,
47 unused-argument,
48 unused-import,
49 unused-variable,
50 useless-super-delegation,
51 nonstandard-exception,
52# "C" Coding convention violations
53 bad-continuation,
54 consider-iterating-dictionary,
55 consider-using-enumerate,
56 invalid-name,
57 len-as-condition,
58 misplaced-comparison-constant,
59 missing-docstring,
60 singleton-comparison,
61 superfluous-parens,
62 ungrouped-imports,
63 wrong-import-order,
64# "R" Refactor recommendations
65 abstract-class-little-used,
66 abstract-class-not-used,
67 consider-merging-isinstance,
68 consider-using-ternary,
69 duplicate-code,
70 interface-not-implemented,
71 no-else-return,
72 no-self-use,
73 redefined-argument-from-local,
74 simplifiable-if-statement,
75 too-few-public-methods,
76 too-many-ancestors,
77 too-many-arguments,
78 too-many-branches,
79 too-many-instance-attributes,
80 too-many-lines,
81 too-many-locals,
82 too-many-nested-blocks,
83 too-many-public-methods,
84 too-many-return-statements,
85 too-many-statements
86
87[BASIC]
88# Variable names can be 1 to 31 characters long, with lowercase and underscores
89variable-rgx=[a-z_][a-z0-9_]{0,30}$
90
91# Argument names can be 2 to 31 characters long, with lowercase and underscores
92argument-rgx=[a-z_][a-z0-9_]{1,30}$
93
94# Method names should be at least 3 characters long
95# and be lowercased with underscores
96method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
97
98# Don't require docstrings on tests.
99no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
100
101[FORMAT]
102# Maximum number of characters on a single line.
103max-line-length=79
104
105[VARIABLES]
106# List of additional names supposed to be defined in builtins. Remember that
107# you should avoid to define new builtins when possible.
108# _ is used by our localization
109additional-builtins=_
110
111[CLASSES]
112# List of interface methods to ignore, separated by a comma.
113ignore-iface-methods=
114
115[IMPORTS]
116# Deprecated modules which should not be used, separated by a comma
117deprecated-modules=
118# should use oslo_serialization.jsonutils
119 json
120
121[TYPECHECK]
122# List of module names for which member attributes should not be checked
123ignored-modules=six.moves,_MovedItems
124
125[REPORTS]
126# Tells whether to display a full report or only the messages
127reports=no
diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000..16dd3e6
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,21 @@
1# Copyright 2018 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15- project:
16 check:
17 jobs:
18 - tox-pep8
19 gate:
20 jobs:
21 - tox-pep8
diff --git a/oslogmerger/oslogmerger.py b/oslogmerger/oslogmerger.py
index c57750c..1344e62 100644
--- a/oslogmerger/oslogmerger.py
+++ b/oslogmerger/oslogmerger.py
@@ -1,8 +1,6 @@
1from __future__ import print_function 1from __future__ import print_function
2import argparse 2import argparse
3from datetime import datetime, timedelta 3from datetime import datetime, timedelta
4import dateutil.parser
5import dateutil.tz
6import hashlib 4import hashlib
7import heapq 5import heapq
8import itertools 6import itertools
@@ -12,9 +10,11 @@ import sys
12import tempfile 10import tempfile
13import time 11import time
14 12
15from six.moves.urllib.parse import urlparse 13import dateutil.parser
14import dateutil.tz
15from six.moves.urllib.request import urlopen
16 16
17__version__ = '1.1.0' 17__version__ = '1.1.1'
18 18
19EXTRALINES_PADDING = " " * 40 19EXTRALINES_PADDING = " " * 40
20CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir() 20CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir()
@@ -62,7 +62,6 @@ FILE_MAP = {
62 'lbaas-agent': 'LBAAS', 62 'lbaas-agent': 'LBAAS',
63 'metadata-agent': 'META', 63 'metadata-agent': 'META',
64 'metering-agent': 'MTR', 64 'metering-agent': 'MTR',
65 'openvswitch-agent': 'VSWI',
66 'server': 'API', 65 'server': 'API',
67 'linuxbridge-agent': 'SVC', 66 'linuxbridge-agent': 'SVC',
68 'netprobe': 'NET', 67 'netprobe': 'NET',
@@ -77,7 +76,6 @@ FILE_MAP = {
77 'ceilometer-dbsync': 'DBSY', 76 'ceilometer-dbsync': 'DBSY',
78 'central': 'CENT', 77 'central': 'CENT',
79 'collector': 'COLL', 78 'collector': 'COLL',
80 'compute': 'CPT',
81} 79}
82 80
83 81
@@ -175,14 +173,14 @@ class LibvirtdParser(LogParser):
175 This parser handles libvirtd.log and libvirt domain logs. Domain logs 173 This parser handles libvirtd.log and libvirt domain logs. Domain logs
176 contain a mixture of libvirt and qemu logs, hence the 2 log formats. 174 contain a mixture of libvirt and qemu logs, hence the 2 log formats.
177 """ 175 """
178 LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date 176 LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) ' # Date
179 '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time 177 '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time
180 '(' # 178 '('
181 '([+-])(\d{2})(\d{2})' # Timezone 179 '([+-])(\d{2})(\d{2})' # Timezone
182 '):\s*') # 180 '):\s*')
183 181
184 QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date 182 QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T' # Date
185 '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time 183 '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time
186 184
187 def parse_line(self, line): 185 def parse_line(self, line):
188 m = self.LIBVIRT.match(line) 186 m = self.LIBVIRT.match(line)
@@ -248,11 +246,11 @@ class RawSyslog(LogParser):
248 # manually anyway. 246 # manually anyway.
249 HEADER = re.compile('<\d+>\d+\s' 247 HEADER = re.compile('<\d+>\d+\s'
250 '(' 248 '('
251 '(\d{4})-(\d{2})-(\d{2})T' # Date 249 '(\d{4})-(\d{2})-(\d{2})T' # Date
252 '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time 250 '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time
253 '(' # 251 '('
254 '([+-])(\d{2}):(\d{2})' # Timezone 252 '([+-])(\d{2}):(\d{2})' # Timezone
255 ')' # 253 ')'
256 ')\s*') 254 ')\s*')
257 255
258 def parse_line(self, line): 256 def parse_line(self, line):
@@ -324,7 +322,7 @@ class TSLogParser(LogParser):
324 def parse_line(self, line): 322 def parse_line(self, line):
325 end, timestamp = self._read_timestamp(line) 323 end, timestamp = self._read_timestamp(line)
326 dt = self.start_date + timedelta(seconds=timestamp) 324 dt = self.start_date + timedelta(seconds=timestamp)
327 dt = dt.replace(tzinfo = self.cfg.default_tz) 325 dt = dt.replace(tzinfo=self.cfg.default_tz)
328 return dt, line[:end + 1], line[end + 1:] 326 return dt, line[:end + 1], line[end + 1:]
329 327
330 328
@@ -453,7 +451,6 @@ DETECTED_LOG_TYPES = [
453 451
454 452
455def process_logs(cfg): 453def process_logs(cfg):
456 filename_alias = {}
457 logs = [] 454 logs = []
458 455
459 paths_aliases = {} 456 paths_aliases = {}
diff --git a/oslogmerger/probes/netprobe.py b/oslogmerger/probes/netprobe.py
index 0a40138..f1f1305 100644
--- a/oslogmerger/probes/netprobe.py
+++ b/oslogmerger/probes/netprobe.py
@@ -88,7 +88,7 @@ def spawn_tcpdump(interface, netns=None,
88 reads, writes, excs = select.select([tcpdump.stdout], [], []) 88 reads, writes, excs = select.select([tcpdump.stdout], [], [])
89 try: 89 try:
90 out = reads[0].readline() 90 out = reads[0].readline()
91 except: 91 except Exception:
92 continue 92 continue
93 if out == '': 93 if out == '':
94 break 94 break
@@ -187,5 +187,6 @@ def main():
187 output = open(args.output_file, 'w', 0) 187 output = open(args.output_file, 'w', 0)
188 scan_loop(args) 188 scan_loop(args)
189 189
190
190if __name__ == '__main__': 191if __name__ == '__main__':
191 main() 192 main()
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d87e947
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
1python-dateutil
2six
diff --git a/setup.cfg b/setup.cfg
index af57bc0..876a791 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1 +1,35 @@
1[bdist_wheel] 1[metadata]
2name = os-log-merger
3summary = OpenStack Log merge tool
4description-file =
5 README.rst
6author = OpenStack
7author-email = openstack-dev@lists.openstack.org
8home-page = https://github.com/openstack/os-log-merger
9classifier =
10 Environment :: OpenStack
11 Intended Audience :: Developers
12 Intended Audience :: System Administrators
13 License :: OSI Approved :: Apache Software License
14 Operating System :: POSIX :: Linux
15 Programming Language :: Python
16 Programming Language :: Python :: 2
17 Programming Language :: Python :: 2.7
18 Programming Language :: Python :: 3
19 Programming Language :: Python :: 3.5
20 Topic:: Utilities
21keywords = OpenStack log merger
22license = Apache Software License
23
24[files]
25packages =
26 oslogmerger
27
28[entry_points]
29console_scripts =
30 os-log-merger=oslogmerger.oslogmerger:main
31 oslogmerger=oslogmerger.oslogmerger:main
32 netprobe=oslogmerger.probes.netprobe:main
33
34[wheel]
35universal = 1
diff --git a/setup.py b/setup.py
index 1ca38f9..9c697b9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,71 +1,8 @@
1from setuptools import setup, find_packages 1from setuptools import setup
2from codecs import open
3from os import path
4from oslogmerger.oslogmerger import __version__ 2from oslogmerger.oslogmerger import __version__
5 3
6here = path.abspath(path.dirname(__file__))
7
8# Get the long description from the README file
9with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
10 long_description = f.read()
11
12setup( 4setup(
13 name='os-log-merger',
14
15 version=__version__, 5 version=__version__,
16 description='OpenStack Log merge tool', 6 setup_requires=['pbr>=2.0.0'],
17 long_description=long_description, 7 pbr=True,
18
19 # The project's main homepage.
20 url='https://github.com/mangelajo/os-log-merger',
21
22 # Author details
23 author='Miguel Angel Ajo, Gorka Eguileor',
24 author_email='majopela@redhat.com, geguileo@redhat.com',
25
26 # Choose your license
27 license='Apache Software License',
28
29 # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
30 classifiers=[
31 # How mature is this project? Common values are
32 # 3 - Alpha
33 # 4 - Beta
34 # 5 - Production/Stable
35 'Development Status :: 3 - Alpha',
36
37 # Indicate who your project is intended for
38 'Intended Audience :: Developers',
39
40 # Pick your license as you wish (should match "license" above)
41 'License :: OSI Approved :: Apache Software License',
42 'Topic :: Utilities',
43 'Environment :: OpenStack',
44
45 # Specify the Python versions you support here. In particular, ensure
46 # that you indicate whether you support Python 2, Python 3 or both.
47 'Programming Language :: Python :: 2.7',
48 ],
49
50 # What does your project relate to?
51 keywords='OpenStack log merger',
52
53 # You can just specify the packages manually here if your project is
54 # simple. Or you can use find_packages().
55 packages=find_packages(exclude=['contrib', 'docs', 'tests']),
56
57 py_modules=['oslogmerger'],
58 install_requires=[],
59 # extras_require={
60 # 'dev': ['check-manifest'],
61 # 'test': ['coverage'],
62 # },
63
64 entry_points={
65 'console_scripts': [
66 'os-log-merger=oslogmerger.oslogmerger:main',
67 'oslogmerger=oslogmerger.oslogmerger:main',
68 'netprobe=oslogmerger.probes.netprobe:main',
69 ],
70 },
71) 8)
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..9154faf
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1,11 @@
1# The order of packages is significant, because pip processes them in the order
2# of appearance. Changing the order has an impact on the overall integration
3# process, which may cause wedges in the gate later.
4flake8
5flake8-import-order==0.12 # LGPLv3
6mock>=2.0.0 # BSD
7python-subunit>=0.0.18 # Apache-2.0/BSD
8testtools>=1.4.0 # MIT
9oslotest>=1.10.0 # Apache-2.0
10os-testr>=1.0.0 # Apache-2.0
11pylint==1.4.5 # GPLv2
diff --git a/tools/coding-checks.sh b/tools/coding-checks.sh
new file mode 100755
index 0000000..0985ec8
--- /dev/null
+++ b/tools/coding-checks.sh
@@ -0,0 +1,58 @@
1#!/bin/sh
2
3set -eu
4
5usage () {
6 echo "Usage: $0 [OPTION]..."
7 echo "Run os-log-mergers's coding check(s)"
8 echo ""
9 echo " -Y, --pylint [<basecommit>] Run pylint check on the entire oslogmerger module or just files changed in basecommit (e.g. HEAD~1)"
10 echo " -h, --help Print this usage message"
11 echo
12 exit 0
13}
14
15process_options () {
16 i=1
17 while [ $i -le $# ]; do
18 eval opt=\$$i
19 case $opt in
20 -h|--help) usage;;
21 -Y|--pylint) pylint=1;;
22 *) scriptargs="$scriptargs $opt"
23 esac
24 i=$((i+1))
25 done
26}
27
28run_pylint () {
29 local target="${scriptargs:-all}"
30
31 if [ "$target" = "all" ]; then
32 files="oslogmerger"
33 else
34 case "$target" in
35 *HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");;
36 *) echo "$target is an unrecognized basecommit"; exit 1;;
37 esac
38 fi
39
40 echo "Running pylint..."
41 echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..."
42 if [ -n "${files}" ]; then
43 pylint --rcfile=.pylintrc --output-format=colorized ${files}
44 else
45 echo "No python changes in this commit, pylint check not required."
46 exit 0
47 fi
48}
49
50scriptargs=
51pylint=1
52
53process_options $@
54
55if [ $pylint -eq 1 ]; then
56 run_pylint
57 exit 0
58fi
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..0fdbad7
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,59 @@
1[tox]
2envlist = py35,pep8
3minversion = 2.3.2
4skipsdist = True
5
6[testenv]
7setenv = VIRTUAL_ENV={envdir}
8 PYTHONWARNINGS=default::DeprecationWarning
9passenv = TRACE_FAILONLY GENERATE_HASHES http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
10usedevelop = True
11install_command =
12 pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
13deps = -r{toxinidir}/requirements.txt
14 -r{toxinidir}/test-requirements.txt
15whitelist_externals = sh
16commands =
17 {toxinidir}/tools/ostestr_compat_shim.sh {posargs}
18# there is also secret magic in ostestr which lets you run in a fail only
19# mode. To do this define the TRACE_FAILONLY environmental variable.
20
21[testenv:common]
22# Fake job to define environment variables shared between dsvm/non-dsvm jobs
23setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
24commands = false
25
26[testenv:pep8]
27basepython = python2.7
28deps =
29 {[testenv]deps}
30commands=
31 flake8
32 sh ./tools/coding-checks.sh --pylint '{posargs}'
33whitelist_externals =
34 sh
35 bash
36
37[testenv:venv]
38commands = {posargs}
39
40[flake8]
41# E125 continuation line does not distinguish itself from next logical line
42# E126 continuation line over-indented for hanging indent
43# E128 continuation line under-indented for visual indent
44# E129 visually indented line with same indent as next logical line
45# E265 block comment should start with '# '
46# H404 multi line docstring should start with a summary
47# H405 multi line docstring summary not separated with an empty line
48# N530 direct neutron imports not allowed
49# TODO(ihrachys) figure out what to do with N534 and N536
50# N534 Untranslated exception message
51# N536 Use assertIsNone rather than assertEqual to check for None values
52ignore = E125,E126,E128,E129,E265,H404,H405,N530,N534,N536
53# H106: Don't put vim configuration in source files
54# H203: Use assertIs(Not)None to check for None
55# H904: Delay string interpolations at logging calls
56enable-extensions=H106,H203,H904
57show-source = true
58exclude = ./.*,build,dist,doc
59import-order-style = pep8