Setup tox.ini for unit and pep8 testing

This patch includes fixes to all the pep8 warnings.

Change-Id: I4f87e1428258824509d2d83879312a1f0dff0a8a

.pylintrc

@@ -0,0 +1,127 @@
+# The format of this file isn't really documented; just use --generate-rcfile
+[MASTER]
+# Add <file or directory> to the black list. It should be a base name, not a
+# path. You may set this option multiple times.
+ignore=.git,tests
+
+[MESSAGES CONTROL]
+# NOTE(gus): This is a long list.  A number of these are important and
+# should be re-enabled once the offending code is fixed (or marked
+# with a local disable)
+disable=
+# "F" Fatal errors that prevent further processing
+ import-error,
+# "I" Informational noise
+ locally-disabled,
+# "E" Error for important programming issues (likely bugs)
+ access-member-before-definition,
+ anomalous-backslash-in-string,
+ no-member,
+ no-method-argument,
+ no-self-argument,
+ not-an-iterable,
+# "W" Warnings for stylistic problems or minor programming issues
+ abstract-method,
+ arguments-differ,
+ attribute-defined-outside-init,
+ bad-builtin,
+ bad-indentation,
+ broad-except,
+ dangerous-default-value,
+ deprecated-lambda,
+ expression-not-assigned,
+ fixme,
+ global-statement,
+ literal-comparison,
+ no-init,
+ non-parent-init-called,
+ not-callable,
+ protected-access,
+ redefined-builtin,
+ redefined-outer-name,
+ signature-differs,
+ star-args,
+ super-init-not-called,
+ super-on-old-class,
+ unpacking-non-sequence,
+ unused-argument,
+ unused-import,
+ unused-variable,
+ useless-super-delegation,
+ nonstandard-exception,
+# "C" Coding convention violations
+ bad-continuation,
+ consider-iterating-dictionary,
+ consider-using-enumerate,
+ invalid-name,
+ len-as-condition,
+ misplaced-comparison-constant,
+ missing-docstring,
+ singleton-comparison,
+ superfluous-parens,
+ ungrouped-imports,
+ wrong-import-order,
+# "R" Refactor recommendations
+ abstract-class-little-used,
+ abstract-class-not-used,
+ consider-merging-isinstance,
+ consider-using-ternary,
+ duplicate-code,
+ interface-not-implemented,
+ no-else-return,
+ no-self-use,
+ redefined-argument-from-local,
+ simplifiable-if-statement,
+ too-few-public-methods,
+ too-many-ancestors,
+ too-many-arguments,
+ too-many-branches,
+ too-many-instance-attributes,
+ too-many-lines,
+ too-many-locals,
+ too-many-nested-blocks,
+ too-many-public-methods,
+ too-many-return-statements,
+ too-many-statements
+
+[BASIC]
+# Variable names can be 1 to 31 characters long, with lowercase and underscores
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
+
+# Argument names can be 2 to 31 characters long, with lowercase and underscores
+argument-rgx=[a-z_][a-z0-9_]{1,30}$
+
+# Method names should be at least 3 characters long
+# and be lowercased with underscores
+method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
+
+# Don't require docstrings on tests.
+no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=79
+
+[VARIABLES]
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+# _ is used by our localization
+additional-builtins=_
+
+[CLASSES]
+# List of interface methods to ignore, separated by a comma.
+ignore-iface-methods=
+
+[IMPORTS]
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=
+# should use oslo_serialization.jsonutils
+ json
+
+[TYPECHECK]
+# List of module names for which member attributes should not be checked
+ignored-modules=six.moves,_MovedItems
+
+[REPORTS]
+# Tells whether to display a full report or only the messages
+reports=no

.zuul.yaml

@@ -0,0 +1,21 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- project:
+    check:
+      jobs:
+        - tox-pep8
+    gate:
+      jobs:
+        - tox-pep8

oslogmerger/oslogmerger.py

@@ -1,8 +1,6 @@
 from __future__ import print_function
 import argparse
 from datetime import datetime, timedelta
-import dateutil.parser
-import dateutil.tz
 import hashlib
 import heapq
 import itertools
@@ -12,9 +10,11 @@ import sys
 import tempfile
 import time
 
-from six.moves.urllib.parse import urlparse
+import dateutil.parser
+import dateutil.tz
+from six.moves.urllib.request import urlopen
 
-__version__ = '1.1.0'
+__version__ = '1.1.1'
 
 EXTRALINES_PADDING = " " * 40
 CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir()
@@ -62,7 +62,6 @@ FILE_MAP = {
     'lbaas-agent': 'LBAAS',
     'metadata-agent': 'META',
     'metering-agent': 'MTR',
-    'openvswitch-agent': 'VSWI',
     'server': 'API',
     'linuxbridge-agent': 'SVC',
     'netprobe': 'NET',
@@ -77,7 +76,6 @@ FILE_MAP = {
     'ceilometer-dbsync': 'DBSY',
     'central': 'CENT',
     'collector': 'COLL',
-    'compute': 'CPT',
 }
 
 
@@ -175,14 +173,14 @@ class LibvirtdParser(LogParser):
     This parser handles libvirtd.log and libvirt domain logs. Domain logs
     contain a mixture of libvirt and qemu logs, hence the 2 log formats.
     """
-    LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) '         # Date
+    LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) '  # Date
-                         '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time
+                         '(\d{2}):(\d{2}):(\d{2})\.(\d{3})'  # Time
-                         '('                                #
+                         '('
-                          '([+-])(\d{2})(\d{2})'            # Timezone
+                         '([+-])(\d{2})(\d{2})'  # Timezone
-                         '):\s*')                           #
+                         '):\s*')
 
-    QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T'            # Date
+    QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T'  # Date
-                      '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time
+                      '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*')  # Time
 
     def parse_line(self, line):
         m = self.LIBVIRT.match(line)
@@ -248,11 +246,11 @@ class RawSyslog(LogParser):
     # manually anyway.
     HEADER = re.compile('<\d+>\d+\s'
                         '('
-                         '(\d{4})-(\d{2})-(\d{2})T'       # Date
+                        '(\d{4})-(\d{2})-(\d{2})T'  # Date
-                         '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time
+                        '(\d{2}):(\d{2}):(\d{2})\.(\d+)'  # Time
-                         '('                              #
+                        '('
-                          '([+-])(\d{2}):(\d{2})'         # Timezone
+                        '([+-])(\d{2}):(\d{2})'  # Timezone
-                         ')'                              #
+                        ')'
                         ')\s*')
 
     def parse_line(self, line):
@@ -324,7 +322,7 @@ class TSLogParser(LogParser):
     def parse_line(self, line):
         end, timestamp = self._read_timestamp(line)
         dt = self.start_date + timedelta(seconds=timestamp)
-        dt = dt.replace(tzinfo = self.cfg.default_tz)
+        dt = dt.replace(tzinfo=self.cfg.default_tz)
         return dt, line[:end + 1], line[end + 1:]
 
 
@@ -453,7 +451,6 @@ DETECTED_LOG_TYPES = [
 
 
 def process_logs(cfg):
-    filename_alias = {}
     logs = []
 
     paths_aliases = {}

oslogmerger/probes/netprobe.py

@@ -88,7 +88,7 @@ def spawn_tcpdump(interface, netns=None,
         reads, writes, excs = select.select([tcpdump.stdout], [], [])
         try:
             out = reads[0].readline()
-        except:
+        except Exception:
             continue
         if out == '':
             break
@@ -187,5 +187,6 @@ def main():
     output = open(args.output_file, 'w', 0)
     scan_loop(args)
 
+
 if __name__ == '__main__':
     main()

requirements.txt

@@ -0,0 +1,2 @@
+python-dateutil
+six

setup.cfg

@@ -1 +1,35 @@
-[bdist_wheel]
+[metadata]
+name = os-log-merger
+summary = OpenStack Log merge tool
+description-file =
+    README.rst
+author = OpenStack
+author-email = openstack-dev@lists.openstack.org
+home-page = https://github.com/openstack/os-log-merger
+classifier =
+    Environment :: OpenStack
+    Intended Audience :: Developers
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.5
+    Topic:: Utilities
+keywords = OpenStack log merger
+license = Apache Software License
+
+[files]
+packages =
+    oslogmerger
+
+[entry_points]
+console_scripts =
+    os-log-merger=oslogmerger.oslogmerger:main
+    oslogmerger=oslogmerger.oslogmerger:main
+    netprobe=oslogmerger.probes.netprobe:main
+
+[wheel]
+universal = 1

setup.py

@@ -1,71 +1,8 @@
-from setuptools import setup, find_packages
+from setuptools import setup
-from codecs import open
-from os import path
 from oslogmerger.oslogmerger import __version__
 
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
-    long_description = f.read()
-
 setup(
-    name='os-log-merger',
-
     version=__version__,
-    description='OpenStack Log merge tool',
+    setup_requires=['pbr>=2.0.0'],
-    long_description=long_description,
+    pbr=True,
-
-    # The project's main homepage.
-    url='https://github.com/mangelajo/os-log-merger',
-
-    # Author details
-    author='Miguel Angel Ajo, Gorka Eguileor',
-    author_email='majopela@redhat.com, geguileo@redhat.com',
-
-    # Choose your license
-    license='Apache Software License',
-
-    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
-    classifiers=[
-        # How mature is this project? Common values are
-        #   3 - Alpha
-        #   4 - Beta
-        #   5 - Production/Stable
-        'Development Status :: 3 - Alpha',
-
-        # Indicate who your project is intended for
-        'Intended Audience :: Developers',
-
-        # Pick your license as you wish (should match "license" above)
-        'License :: OSI Approved :: Apache Software License',
-        'Topic :: Utilities',
-        'Environment :: OpenStack',
-
-        # Specify the Python versions you support here. In particular, ensure
-        # that you indicate whether you support Python 2, Python 3 or both.
-        'Programming Language :: Python :: 2.7',
-    ],
-
-    # What does your project relate to?
-    keywords='OpenStack log merger',
-
-    # You can just specify the packages manually here if your project is
-    # simple. Or you can use find_packages().
-    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
-
-    py_modules=['oslogmerger'],
-    install_requires=[],
-    # extras_require={
-    #    'dev': ['check-manifest'],
-    #    'test': ['coverage'],
-    # },
-
-    entry_points={
-        'console_scripts': [
-            'os-log-merger=oslogmerger.oslogmerger:main',
-            'oslogmerger=oslogmerger.oslogmerger:main',
-	    'netprobe=oslogmerger.probes.netprobe:main',
-        ],
-    },
 )

test-requirements.txt

@@ -0,0 +1,11 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+flake8
+flake8-import-order==0.12 # LGPLv3
+mock>=2.0.0 # BSD
+python-subunit>=0.0.18 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT
+oslotest>=1.10.0 # Apache-2.0
+os-testr>=1.0.0 # Apache-2.0
+pylint==1.4.5 # GPLv2

tools/coding-checks.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+set -eu
+
+usage () {
+    echo "Usage: $0 [OPTION]..."
+    echo "Run os-log-mergers's coding check(s)"
+    echo ""
+    echo "  -Y, --pylint [<basecommit>] Run pylint check on the entire oslogmerger module or just files changed in basecommit (e.g. HEAD~1)"
+    echo "  -h, --help                  Print this usage message"
+    echo
+    exit 0
+}
+
+process_options () {
+    i=1
+    while [ $i -le $# ]; do
+        eval opt=\$$i
+        case $opt in
+            -h|--help) usage;;
+            -Y|--pylint) pylint=1;;
+            *) scriptargs="$scriptargs $opt"
+        esac
+        i=$((i+1))
+    done
+}
+
+run_pylint () {
+    local target="${scriptargs:-all}"
+
+    if [ "$target" = "all" ]; then
+        files="oslogmerger"
+    else
+        case "$target" in
+            *HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");;
+            *) echo "$target is an unrecognized basecommit"; exit 1;;
+        esac
+    fi
+
+    echo "Running pylint..."
+    echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..."
+    if [ -n "${files}" ]; then
+        pylint --rcfile=.pylintrc --output-format=colorized ${files}
+    else
+        echo "No python changes in this commit, pylint check not required."
+        exit 0
+    fi
+}
+
+scriptargs=
+pylint=1
+
+process_options $@
+
+if [ $pylint -eq 1 ]; then
+    run_pylint
+    exit 0
+fi

tox.ini

@@ -0,0 +1,59 @@
+[tox]
+envlist = py35,pep8
+minversion = 2.3.2
+skipsdist = True
+
+[testenv]
+setenv = VIRTUAL_ENV={envdir}
+         PYTHONWARNINGS=default::DeprecationWarning
+passenv = TRACE_FAILONLY GENERATE_HASHES http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+usedevelop = True
+install_command =
+    pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+whitelist_externals = sh
+commands =
+  {toxinidir}/tools/ostestr_compat_shim.sh {posargs}
+# there is also secret magic in ostestr which lets you run in a fail only
+# mode. To do this define the TRACE_FAILONLY environmental variable.
+
+[testenv:common]
+# Fake job to define environment variables shared between dsvm/non-dsvm jobs
+setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
+commands = false
+
+[testenv:pep8]
+basepython = python2.7
+deps =
+  {[testenv]deps}
+commands=
+  flake8
+  sh ./tools/coding-checks.sh --pylint '{posargs}'
+whitelist_externals =
+  sh
+  bash
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+# E125 continuation line does not distinguish itself from next logical line
+# E126 continuation line over-indented for hanging indent
+# E128 continuation line under-indented for visual indent
+# E129 visually indented line with same indent as next logical line
+# E265 block comment should start with '# '
+# H404 multi line docstring should start with a summary
+# H405 multi line docstring summary not separated with an empty line
+# N530 direct neutron imports not allowed
+# TODO(ihrachys) figure out what to do with N534 and N536
+# N534 Untranslated exception message
+# N536 Use assertIsNone rather than assertEqual to check for None values
+ignore = E125,E126,E128,E129,E265,H404,H405,N530,N534,N536
+# H106: Don't put vim configuration in source files
+# H203: Use assertIs(Not)None to check for None
+# H904: Delay string interpolations at logging calls
+enable-extensions=H106,H203,H904
+show-source = true
+exclude = ./.*,build,dist,doc
+import-order-style = pep8