Fix tox settings and relevant bugs

I found problem as follows in current tox.ini

- test-requirements can not be read correctly
- some os level command is not in white_list
- one rst file does not follow rst syntac
- fix lack of gitreview file

Purpose of this review request is fixing above problems.

Change-Id: I1d2b29bbc7cdc506f5187c2ef6ee03fd2b908f58

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=openstack/slogging.git
+defaultbranch=master

bindep.txt

@@ -0,0 +1,6 @@
+# This is a cross-platform list tracking distribution packages needed by tests;
+# see http://docs.openstack.org/infra/bindep/ for additional information.
+
+liberasurecode-dev [platform:dpkg]
+# There's no library in CentOS 7 but Fedora and openSUSE have it.
+liberasurecode-devel [platform:rpm !platform:centos]

doc/requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-openstackdocstheme>=1.18.1 # Apache-2.0
-sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD
-reno>=2.5.0 # Apache-2.0
-sphinxcontrib-httpdomain>=1.3.0 # BSD
+openstackdocstheme # Apache-2.0
+sphinx!=1.6.6,!=1.6.7 # BSD
+reno # Apache-2.0
+sphinxcontrib-httpdomain # BSD

doc/source/install/build_rpm_packages.rst

@@ -4,18 +4,29 @@ How to Build to RPM Packages
 
 #. Make sure you have rpm-build installed::
 
- - on Fedora/CentOS/RedHat::
-
     sudo yum install rpm-build
 
- - on OpenSUSE::
 
-    sudo zypper install rpm-build
+Fedora/CentOS/RedHat
+~~~~~~~~~~~~~~~~~~~~
 
-#. Thsn type following command at the top of slogging directory::
+
+#. Type following command at the top of slogging directory::
 
     sudo python setup.py bdist_rpm
 
 #. Check if the RPM package has built::
 
     ls dist/slogging-[slogging-version]
+
+
+OpenSUSE
+~~~~~~~~
+
+#. Type following command at the top of slogging directory::
+
+    sudo zypper install rpm-build
+
+#. Check if the RPM package has built::
+
+    ls dist/slogging-[slogging-version]

requirements.txt

@@ -1,4 +1,6 @@
-pbr!=2.1.0,>=2.0.0 # Apache-2.0
-iptools>=0.4
-tzlocal
-swift>=2.14.0
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+pbr!=2.1.0 # Apache-2.0
+pytz
+netaddr

setup.py

@@ -1,5 +1,4 @@
-#!/usr/bin/python
-# Copyright (c) 2010-2011 OpenStack, LLC.
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,15 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
 try:
     import multiprocessing  # noqa
 except ImportError:
     pass
 
-
 setuptools.setup(
-    setup_requires=['pbr'],
-    pbr=True,
-)
+    setup_requires=['pbr>=2.0.0'],
+    pbr=True)

slogging/access_processor.py

@@ -16,20 +16,19 @@
 from datetime import datetime
 import pytz
 from slogging import common
-from tzlocal import get_localzone
 from urllib import unquote
 from urlparse import urlparse
 
 # conditionalize the return_ips method based on whether or not iptools
 # is present in the system. Without iptools, you will lack CIDR support.
 try:
-    from iptools import IpRangeList
+    from netaddr import IPSet
     CIDR_support = True
 
     def return_ips(conf, conf_tag):
-        return set(k for k in IpRangeList(*[
+        return IPSet([
             x.strip() for x in conf.get(conf_tag, '').split(',')
-            if x.strip()]))
+            if x.strip()])
 
     def sanitize_ips(line_data):
         for x in ['lb_ip', 'client_ip', 'log_source']:
@@ -48,7 +47,7 @@ from swift.common import utils
 month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
 LISTING_PARAMS = set(
     'path limit format delimiter marker end_marker prefix'.split())
-local_zone = get_localzone()
+local_zone = common.get_localzone()
 
 
 class AccessLogProcessor(object):
@@ -63,7 +62,7 @@ class AccessLogProcessor(object):
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
         self.logger = utils.get_logger(conf, log_route='access-processor')
         self.time_zone = common.get_time_zone(conf, self.logger, 'time_zone',
-                                              str(local_zone))
+                                              local_zone)
 
     def log_line_parser(self, raw_log):
         '''given a raw access log line, return a dict of the good parts'''

slogging/common.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import pytz
 
 
@@ -44,3 +45,14 @@ def get_format_type(conf, logger, key, default):
         logger.warning(msg)
         format_type = default
     return format_type
+
+
+def get_localzone():
+    """Get time_zone value."""
+    try:
+        str_time_zone = '/'.join(
+            os.readlink('/etc/localtime').split('/')[-2:])
+    except Exception:
+        str_time_zone = 'UTC'
+
+    return str_time_zone

slogging/db_stats_collector.py

@@ -29,11 +29,10 @@ from swift.container.backend import ContainerBroker
 from swift.container.server import DATADIR as container_server_data_dir
 
 import time
-from tzlocal import get_localzone
 import urllib
 
 
-local_zone = get_localzone()
+local_zone = common.get_localzone()
 
 
 class DatabaseStatsCollector(Daemon):
@@ -57,7 +56,7 @@ class DatabaseStatsCollector(Daemon):
         self.logger = utils.get_logger(stats_conf,
                                        log_route='%s-stats' % stats_type)
         self.time_zone = common.get_time_zone(stats_conf, self.logger,
-                                              'time_zone', str(local_zone))
+                                              'time_zone', local_zone)
 
     def run_once(self, *args, **kwargs):
         self.logger.info(_("Gathering %s stats") % self.stats_type)

slogging/log_processor.py

@@ -25,11 +25,10 @@ from slogging import log_common
 from swift.common.daemon import Daemon
 from swift.common import utils
 import time
-from tzlocal import get_localzone
 
 
 now = datetime.datetime.now
-local_zone = get_localzone()
+local_zone = common.get_localzone()
 
 
 class LogProcessor(log_common.LogProcessorCommon):
@@ -137,7 +136,7 @@ class LogProcessorDaemon(Daemon):
         self._keylist_mapping = None
         self.processed_files_filename = 'processed_files.pickle.gz'
         self.time_zone = common.get_time_zone(c, self.logger, 'time_zone',
-                                              str(local_zone))
+                                              local_zone)
         self.format_type = common.get_format_type(c, self.logger,
                                                   'format_type', 'csv')
 

test-requirements.txt

@@ -3,11 +3,12 @@
 # process, which may cause wedges in the gate later.
 
 # Hacking already pins down pep8, pyflakes and flake8
-hacking>=0.11.0,<0.12 # Apache-2.0
-coverage>=3.6 # Apache-2.0
+hacking<0.12,>=0.11.0 # Apache-2.0
+coverage!=4.4 # Apache-2.0
 nose # LGPL
 nosexcover # BSD
-nosehtmloutput>=0.0.3 # Apache-2.0
+nosehtmloutput # Apache-2.0
+flake8<=2.5.5
 
 # Security checks
 bandit>=1.1.0 # Apache-2.0

test/speed/test_cidr_speed.py

@@ -15,7 +15,7 @@
 
 
 import datetime
-import iptools
+import netaddr
 import unittest
 
 
@@ -25,15 +25,15 @@ class TestAccessProcessorSpeed(unittest.TestCase):
         line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \
                '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \
                '200 - StaticWeb - - 17005 - txn - 0.0095 -'
-        ips1 = iptools.IpRangeList(*[x.strip() for x in
-                                     '127.0.0.1,192.168/16,10/24'.split(',')
-                                     if x.strip()])
-        ips2 = iptools.IpRangeList(*[x.strip() for x in
-                                     '172.168/16,10/30'.split(',')
-                                     if x.strip()])
-        ips3 = iptools.IpRangeList(*[x.strip() for x in
-                                     '127.0.0.1,11/24'.split(',')
-                                     if x.strip()])
+        ips1 = netaddr.IPSet([x.strip() for x in
+                              '127.0.0.1,192.168/16,10/24'.split(',')
+                              if x.strip()])
+        ips2 = netaddr.IPSet([x.strip() for x in
+                              '172.168/16,10/30'.split(',')
+                              if x.strip()])
+        ips3 = netaddr.IPSet([x.strip() for x in
+                              '127.0.0.1,11/24'.split(',')
+                              if x.strip()])
 
         orig_start = datetime.datetime.utcnow()
         hit = 0
@@ -49,9 +49,9 @@ class TestAccessProcessorSpeed(unittest.TestCase):
         self.assertEqual(hit, 255)
 
         # now, let's check the speed with sets
-        set1 = set(k for k in ips1)
-        set2 = set(k for k in ips2)
-        set3 = set(k for k in ips3)
+        set1 = ips1
+        set2 = ips2
+        set3 = ips3
 
         new_start = datetime.datetime.utcnow()
         hit = 0

tox.ini

@@ -2,6 +2,7 @@
 envlist = py27,docs,pep8
 
 [testenv]
+usedevelop = True
 setenv = VIRTUAL_ENV={envdir}
          NOSE_WITH_OPENSTACK=1
          NOSE_OPENSTACK_COLOR=1
@@ -13,55 +14,37 @@ deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/re
 commands=
   py27: {[unit_tests]commands}
   py35: {[unit_tests]commands}
-
+whitelist_externals = find
+                      rm
 
 [testenv:pep8]
 deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
+       -r{toxinidir}/test-requirements.txt
 commands = flake8 {posargs}
 
-
-[testenv:cover]
-setenv = VIRTUAL_ENV={envdir}
-         NOSE_WITH_COVERAGE=1
-         NOSE_COVER_HTML_DIR={toxinidir}/cover
-         NOSE_COVER_HTML=1
-         NOSE_COVER_ERASE=1
-commands =
-  {[unit_tests]commands}
-
-
 [testenv:py27]
 basepython = python2.7
 deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
+       -r{toxinidir}/test-requirements.txt
 setenv = SWIFT_TEST_CONFIG_FILE={toxinidir}/test/sample.conf
          SWIFT_PROXY_TEST_CONFIG_FILE={toxinidir}/test/sample.proxy-server.conf
 commands =
   {[unit_tests]commands}
 
-
 [unit_tests]
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 commands = find . -type f -name "*.py[c|o]" -delete
            find . -type d -name "__pycache__" -delete
            nosetests {posargs:test/unit}
 
-
-[testenv:venv]
-commands = {posargs}
-
 [flake8]
 exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,dash_template
 max-complexity = 20
-import-order-style = pep8
-
 
 [testenv:docs]
 deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
        -r{toxinidir}/doc/requirements.txt
 commands = python setup.py build_sphinx
 
-
 [doc8]
 # File extensions to check
 extensions = .rst, .yaml